1use std::sync::Arc;
31
32use rindle::change::SourceChange;
33use rindle::value::{OwnedRow, OwnedValue};
34use serde_json::{json, Map, Value};
35
36use crate::normalize::NormalizedOp;
37use crate::normalize_protocol::{NormalizedBatch, NormalizedHello, TableWireSchema};
38
39pub fn owned_to_json(v: &OwnedValue) -> Value {
43 match v {
44 OwnedValue::Absent => {
49 debug_assert!(false, "OwnedValue::Absent must never reach the server wire");
50 Value::Null
51 }
52 OwnedValue::Null => Value::Null,
53 OwnedValue::Bool(b) => Value::Bool(*b),
54 OwnedValue::Int(i) => json!(*i as f64),
55 OwnedValue::Float(f) => json!(*f),
56 OwnedValue::Str(s) => Value::String(s.to_string()),
57 OwnedValue::Json(s) => Value::String(s.to_string()),
58 }
59}
60
61pub fn json_to_owned(v: &Value) -> OwnedValue {
65 match v {
66 Value::Null => OwnedValue::Null,
67 Value::Bool(b) => OwnedValue::Bool(*b),
68 Value::Number(n) => OwnedValue::Float(n.as_f64().unwrap_or(0.0)),
69 Value::String(s) => OwnedValue::Str(Arc::from(s.as_str())),
70 other => OwnedValue::Str(Arc::from(other.to_string().as_str())),
72 }
73}
74
75pub fn wire_row_to_json(row: &[OwnedValue]) -> Value {
77 Value::Array(row.iter().map(owned_to_json).collect())
78}
79
80pub fn wire_row_from_json(v: &Value) -> Result<Vec<OwnedValue>, String> {
82 Ok(v.as_array()
83 .ok_or("wire row: not an array")?
84 .iter()
85 .map(json_to_owned)
86 .collect())
87}
88
89pub fn normalized_op_to_json(op: &NormalizedOp) -> Value {
92 match op {
93 NormalizedOp::Add { table, row } => {
94 json!({ "table": table.to_string(), "op": "add", "row": wire_row_to_json(row) })
95 }
96 NormalizedOp::Remove { table, row } => {
97 json!({ "table": table.to_string(), "op": "remove", "row": wire_row_to_json(row) })
98 }
99 NormalizedOp::Edit { table, old, new } => json!({
100 "table": table.to_string(),
101 "op": "edit",
102 "old": wire_row_to_json(old),
103 "new": wire_row_to_json(new),
104 }),
105 }
106}
107
108pub fn normalized_op_from_json(v: &Value) -> Result<NormalizedOp, String> {
113 let obj = v.as_object().ok_or("normalized op: not an object")?;
114 let table: Box<str> = obj
115 .get("table")
116 .and_then(Value::as_str)
117 .ok_or("normalized op missing table")?
118 .into();
119 let row = |key: &str| -> Result<Vec<OwnedValue>, String> {
120 wire_row_from_json(
121 obj.get(key)
122 .ok_or_else(|| format!("normalized op missing {key}"))?,
123 )
124 };
125 match obj.get("op").and_then(Value::as_str) {
126 Some("add") => Ok(NormalizedOp::Add {
127 table,
128 row: row("row")?,
129 }),
130 Some("remove") => Ok(NormalizedOp::Remove {
131 table,
132 row: row("row")?,
133 }),
134 Some("edit") => Ok(NormalizedOp::Edit {
135 table,
136 old: row("old")?,
137 new: row("new")?,
138 }),
139 other => Err(format!("unknown normalized op: {other:?}")),
140 }
141}
142
143pub fn normalized_batch_to_json(b: &NormalizedBatch) -> Value {
147 json!({
148 "epoch": b.epoch as f64,
149 "seq": b.seq as f64,
150 "cv": b.cv as f64,
151 "normalizedFp": format!("{:016x}", b.normalized_fp),
152 "ops": b.ops.iter().map(normalized_op_to_json).collect::<Vec<_>>(),
153 })
154}
155
156pub fn normalized_batch_from_json(v: &Value) -> Result<NormalizedBatch, String> {
158 let obj = v.as_object().ok_or("nbatch: not an object")?;
159 let ops = obj
160 .get("ops")
161 .and_then(Value::as_array)
162 .ok_or("nbatch missing ops")?
163 .iter()
164 .map(normalized_op_from_json)
165 .collect::<Result<Vec<_>, _>>()?;
166 Ok(NormalizedBatch {
167 epoch: counter(obj, "epoch")?,
168 seq: counter(obj, "seq")?,
169 cv: counter(obj, "cv")?,
170 normalized_fp: fingerprint(obj, "normalizedFp")?,
171 ops,
172 })
173}
174
175pub fn normalized_hello_to_json(h: &NormalizedHello) -> Value {
178 let tables: Vec<Value> = h
179 .tables
180 .iter()
181 .map(|t| {
182 json!({
183 "name": t.name.to_string(),
184 "columns": t.columns.iter().map(|c| Value::String(c.to_string())).collect::<Vec<_>>(),
185 "primaryKey": t.primary_key.iter().map(|&i| json!(i as f64)).collect::<Vec<_>>(),
186 })
187 })
188 .collect();
189 json!({
190 "epoch": h.epoch as f64,
191 "comparatorVersion": h.comparator_version as f64,
192 "tables": tables,
193 "normalizedFp": format!("{:016x}", h.normalized_fp),
194 })
195}
196
197pub fn normalized_hello_from_json(v: &Value) -> Result<NormalizedHello, String> {
200 let obj = v.as_object().ok_or("nhello: not an object")?;
201 let tables = obj
202 .get("tables")
203 .and_then(Value::as_array)
204 .ok_or("nhello missing tables")?
205 .iter()
206 .map(table_wire_schema_from_json)
207 .collect::<Result<Vec<_>, _>>()?;
208 let comparator_version = counter(obj, "comparatorVersion")?;
209 let comparator_version = u32::try_from(comparator_version)
210 .map_err(|_| format!("comparatorVersion out of range: {comparator_version}"))?;
211 Ok(NormalizedHello {
212 epoch: counter(obj, "epoch")?,
213 comparator_version,
214 tables,
215 normalized_fp: fingerprint(obj, "normalizedFp")?,
216 })
217}
218
219fn table_wire_schema_from_json(v: &Value) -> Result<TableWireSchema, String> {
220 let obj = v.as_object().ok_or("table schema: not an object")?;
221 let name: Box<str> = obj
222 .get("name")
223 .and_then(Value::as_str)
224 .ok_or("table schema missing name")?
225 .into();
226 let columns = obj
227 .get("columns")
228 .and_then(Value::as_array)
229 .ok_or("table schema missing columns")?
230 .iter()
231 .map(|c| {
232 c.as_str()
233 .map(Box::from)
234 .ok_or_else(|| "table schema: non-string column".to_string())
235 })
236 .collect::<Result<Vec<_>, _>>()?;
237 let primary_key = obj
238 .get("primaryKey")
239 .and_then(Value::as_array)
240 .ok_or("table schema missing primaryKey")?
241 .iter()
242 .map(|i| {
243 let idx = counter_value(i, "primaryKey index")?;
244 u32::try_from(idx).map_err(|_| format!("primaryKey index out of range: {idx}"))
245 })
246 .collect::<Result<Vec<_>, _>>()?;
247 Ok(TableWireSchema {
248 name,
249 columns,
250 primary_key,
251 })
252}
253
254fn counter(obj: &Map<String, Value>, key: &str) -> Result<u64, String> {
259 counter_value(obj.get(key).ok_or_else(|| format!("missing {key}"))?, key)
260}
261
262fn counter_value(v: &Value, what: &str) -> Result<u64, String> {
263 if let Some(u) = v.as_u64() {
264 return Ok(u);
265 }
266 let f = v.as_f64().ok_or_else(|| format!("{what}: not a number"))?;
267 const MAX_SAFE: f64 = 9007199254740992.0; if f.fract() != 0.0 || !(0.0..=MAX_SAFE).contains(&f) {
269 return Err(format!("{what}: not a wire-safe counter: {f}"));
270 }
271 Ok(f as u64)
272}
273
274fn fingerprint(obj: &Map<String, Value>, key: &str) -> Result<u64, String> {
276 let s = obj
277 .get(key)
278 .and_then(Value::as_str)
279 .ok_or_else(|| format!("missing {key}"))?;
280 u64::from_str_radix(s, 16).map_err(|_| format!("{key}: not a hex fingerprint: {s:?}"))
281}
282
283pub fn packed_row_to_json(row: &OwnedRow) -> Value {
286 Value::Array(row.cells().map(|v| owned_to_json(&v.to_owned())).collect())
287}
288
289pub fn row_change_to_json(table: &str, change: &SourceChange) -> Value {
301 match change {
302 SourceChange::Add(row) => {
303 json!({ "table": table, "op": "add", "row": packed_row_to_json(row) })
304 }
305 SourceChange::Remove(row) => {
306 json!({ "table": table, "op": "remove", "old": packed_row_to_json(row) })
307 }
308 SourceChange::Edit { row, old } => json!({
309 "table": table,
310 "op": "edit",
311 "old": packed_row_to_json(old),
312 "row": packed_row_to_json(row),
313 }),
314 }
315}
316
317pub fn fnv1a64(bytes: &[u8]) -> u64 {
321 let mut h: u64 = 0xcbf29ce4_84222325; for &b in bytes {
323 h ^= b as u64;
324 h = h.wrapping_mul(0x0000_0100_0000_01b3); }
326 h
327}
328
329fn write_f64(buf: &mut Vec<u8>, f: f64) {
358 serde_json::to_writer(&mut *buf, &f).expect("Vec<u8> write is infallible");
359}
360
361fn write_json_str(buf: &mut Vec<u8>, s: &str) {
364 serde_json::to_writer(&mut *buf, &s).expect("Vec<u8> write is infallible");
365}
366
367fn write_hex16(buf: &mut Vec<u8>, mut v: u64) {
371 let mut out = [0u8; 16];
372 for slot in out.iter_mut().rev() {
373 let nibble = (v & 0xf) as u8;
374 *slot = if nibble < 10 {
375 b'0' + nibble
376 } else {
377 b'a' + (nibble - 10)
378 };
379 v >>= 4;
380 }
381 buf.extend_from_slice(&out);
382}
383
384fn write_cell(buf: &mut Vec<u8>, v: &OwnedValue) {
386 match v {
387 OwnedValue::Absent => {
388 debug_assert!(false, "OwnedValue::Absent must never reach the server wire");
389 buf.extend_from_slice(b"null");
390 }
391 OwnedValue::Null => buf.extend_from_slice(b"null"),
392 OwnedValue::Bool(true) => buf.extend_from_slice(b"true"),
393 OwnedValue::Bool(false) => buf.extend_from_slice(b"false"),
394 OwnedValue::Int(i) => write_f64(buf, *i as f64),
396 OwnedValue::Float(f) => write_f64(buf, *f),
397 OwnedValue::Str(s) | OwnedValue::Json(s) => write_json_str(buf, s),
398 }
399}
400
401fn write_row(buf: &mut Vec<u8>, row: &[OwnedValue]) {
403 buf.push(b'[');
404 for (i, c) in row.iter().enumerate() {
405 if i > 0 {
406 buf.push(b',');
407 }
408 write_cell(buf, c);
409 }
410 buf.push(b']');
411}
412
413fn write_op(buf: &mut Vec<u8>, op: &NormalizedOp) {
415 match op {
416 NormalizedOp::Add { table, row } => {
418 buf.extend_from_slice(br#"{"op":"add","row":"#);
419 write_row(buf, row);
420 buf.extend_from_slice(br#","table":"#);
421 write_json_str(buf, table);
422 buf.push(b'}');
423 }
424 NormalizedOp::Remove { table, row } => {
425 buf.extend_from_slice(br#"{"op":"remove","row":"#);
426 write_row(buf, row);
427 buf.extend_from_slice(br#","table":"#);
428 write_json_str(buf, table);
429 buf.push(b'}');
430 }
431 NormalizedOp::Edit { table, old, new } => {
433 buf.extend_from_slice(br#"{"new":"#);
434 write_row(buf, new);
435 buf.extend_from_slice(br#","old":"#);
436 write_row(buf, old);
437 buf.extend_from_slice(br#","op":"edit","table":"#);
438 write_json_str(buf, table);
439 buf.push(b'}');
440 }
441 }
442}
443
444fn write_ops(buf: &mut Vec<u8>, ops: &[NormalizedOp]) {
446 buf.push(b'[');
447 for (i, op) in ops.iter().enumerate() {
448 if i > 0 {
449 buf.push(b',');
450 }
451 write_op(buf, op);
452 }
453 buf.push(b']');
454}
455
456pub fn nbatch_frame(client_qid: u64, batch: &NormalizedBatch, ops_fragment: &str) -> String {
461 let mut buf = Vec::with_capacity(ops_fragment.len() + 96);
462 buf.extend_from_slice(br#"{"batch":{"cv":"#);
463 write_f64(&mut buf, batch.cv as f64);
464 buf.extend_from_slice(br#","epoch":"#);
465 write_f64(&mut buf, batch.epoch as f64);
466 buf.extend_from_slice(br#","normalizedFp":""#);
467 write_hex16(&mut buf, batch.normalized_fp);
468 buf.extend_from_slice(br#"","ops":"#);
469 buf.extend_from_slice(ops_fragment.as_bytes());
470 buf.extend_from_slice(br#","seq":"#);
471 write_f64(&mut buf, batch.seq as f64);
472 buf.extend_from_slice(br#"},"queryId":"#);
473 write_f64(&mut buf, client_qid as f64);
474 buf.extend_from_slice(br#","t":"nbatch"}"#);
475 String::from_utf8(buf).expect("nbatch frame is valid UTF-8")
476}
477
478pub fn nbatch_ops_fragment(batch: &NormalizedBatch) -> String {
483 let mut buf = Vec::new();
484 write_ops(&mut buf, &batch.ops);
485 String::from_utf8(buf).expect("ops fragment is valid UTF-8")
486}
487
488#[cfg(test)]
489mod tests {
490 use super::*;
491 use rindle::value::OwnedValue as V;
492
493 fn hello() -> NormalizedHello {
494 NormalizedHello {
495 epoch: 3,
496 comparator_version: 1,
497 tables: vec![
498 TableWireSchema {
499 name: "comment".into(),
500 columns: vec!["id".into(), "issue_id".into(), "body".into()],
501 primary_key: vec![0],
502 },
503 TableWireSchema {
504 name: "issue".into(),
505 columns: vec!["id".into(), "title".into()],
506 primary_key: vec![0, 1],
507 },
508 ],
509 normalized_fp: 0xdead_beef_0102_0304,
510 }
511 }
512
513 fn batch() -> NormalizedBatch {
514 NormalizedBatch {
515 epoch: 3,
516 seq: 7,
517 cv: 41,
518 normalized_fp: 0xdead_beef_0102_0304,
519 ops: vec![
520 NormalizedOp::Add {
521 table: "issue".into(),
522 row: vec![V::Float(1.0), V::str("a")],
523 },
524 NormalizedOp::Edit {
525 table: "issue".into(),
526 old: vec![V::Float(1.0), V::str("a")],
527 new: vec![V::Float(1.0), V::Null],
528 },
529 NormalizedOp::Remove {
530 table: "comment".into(),
531 row: vec![V::Float(9.0), V::Float(1.0), V::Bool(true)],
532 },
533 ],
534 }
535 }
536
537 #[test]
540 fn hello_golden_wire_form() {
541 let json = normalized_hello_to_json(&hello()).to_string();
542 assert_eq!(
543 json,
544 r#"{"comparatorVersion":1.0,"epoch":3.0,"normalizedFp":"deadbeef01020304","tables":[{"columns":["id","issue_id","body"],"name":"comment","primaryKey":[0.0]},{"columns":["id","title"],"name":"issue","primaryKey":[0.0,1.0]}]}"#
545 );
546 }
547
548 #[test]
551 fn hello_and_batch_round_trip() {
552 let h = hello();
553 let hj = normalized_hello_to_json(&h);
554 assert_eq!(normalized_hello_from_json(&hj).unwrap(), h);
555
556 let b = batch();
557 let bj = normalized_batch_to_json(&b);
558 let back = normalized_batch_from_json(&bj).unwrap();
559 assert_eq!(normalized_batch_to_json(&back), bj);
561 assert_eq!(back.epoch, 3);
562 assert_eq!(back.seq, 7);
563 assert_eq!(back.cv, 41);
564 assert_eq!(back.normalized_fp, 0xdead_beef_0102_0304);
565 assert_eq!(back.ops.len(), 3);
566 }
567
568 #[test]
571 fn int_and_json_cells_collapse_to_wire_forms() {
572 let row = vec![V::Int(5), V::Json(Arc::from(r#"{"k":1}"#))];
573 let j = wire_row_to_json(&row);
574 assert_eq!(j.to_string(), r#"[5.0,"{\"k\":1}"]"#);
575 let back = wire_row_from_json(&j).unwrap();
576 assert!(matches!(back[0], V::Float(f) if f == 5.0));
577 assert!(matches!(&back[1], V::Str(s) if &**s == r#"{"k":1}"#));
578 assert_eq!(wire_row_to_json(&back), j);
580 }
581
582 #[test]
587 fn nbatch_frame_matches_json_macro() {
588 let cases = [
589 batch(),
590 NormalizedBatch {
592 epoch: 0,
593 seq: 0,
594 cv: 0,
595 normalized_fp: 0,
596 ops: vec![],
597 },
598 NormalizedBatch {
600 epoch: 2,
601 seq: 9,
602 cv: 100,
603 normalized_fp: 0xff,
604 ops: vec![
605 NormalizedOp::Add {
606 table: "w*ird \"tbl\"\n".into(),
607 row: vec![
608 V::Null,
609 V::Bool(false),
610 V::Int(-7),
611 V::Float(-0.0),
612 V::Float(1.5e300),
613 V::str("héllo \"q\"\t/\\"),
614 V::Json(Arc::from(r#"{"a":[1,2]}"#)),
615 ],
616 },
617 NormalizedOp::Remove {
619 table: "t".into(),
620 row: vec![V::Int(9_007_199_254_740_993)],
621 },
622 ],
623 },
624 ];
625 for (i, b) in cases.iter().enumerate() {
626 let qid = (i as u64) * 7 + 3;
627 let expect = json!({
628 "t": "nbatch",
629 "queryId": qid as f64,
630 "batch": normalized_batch_to_json(b),
631 })
632 .to_string();
633 let got = nbatch_frame(qid, b, &nbatch_ops_fragment(b));
634 assert_eq!(got, expect, "frame {i} diverged");
635 }
636 }
637
638 #[test]
640 fn accepts_js_integer_counters() {
641 let b = normalized_batch_from_json(
642 &serde_json::from_str(
643 r#"{"epoch":1,"seq":0,"cv":12,"normalizedFp":"00000000000000ff","ops":[]}"#,
644 )
645 .unwrap(),
646 )
647 .unwrap();
648 assert_eq!((b.epoch, b.seq, b.cv, b.normalized_fp), (1, 0, 12, 0xff));
649
650 let h = normalized_hello_from_json(
651 &serde_json::from_str(
652 r#"{"epoch":1,"comparatorVersion":1,"tables":[],"normalizedFp":"0"}"#,
653 )
654 .unwrap(),
655 )
656 .unwrap();
657 assert_eq!(h.epoch, 1);
658 assert_eq!(h.comparator_version, 1);
659 }
660
661 #[test]
662 fn malformed_frames_error_loudly() {
663 let cases = [
664 r#"{"epoch":1.5,"seq":0,"cv":0,"normalizedFp":"0","ops":[]}"#,
666 r#"{"epoch":-1,"seq":0,"cv":0,"normalizedFp":"0","ops":[]}"#,
668 r#"{"epoch":1,"seq":0,"cv":0,"normalizedFp":"zz","ops":[]}"#,
670 r#"{"epoch":1,"seq":0,"cv":0,"normalizedFp":12,"ops":[]}"#,
672 r#"{"epoch":1,"seq":0,"normalizedFp":"0","ops":[]}"#,
674 r#"{"epoch":1,"seq":0,"cv":0,"normalizedFp":"0","ops":[{"table":"t","op":"upsert","row":[]}]}"#,
676 r#"{"epoch":1,"seq":0,"cv":0,"normalizedFp":"0","ops":[{"table":"t","op":"edit","old":[]}]}"#,
678 r#"{"epoch":1,"seq":0,"cv":0,"normalizedFp":"0","ops":[{"table":"t","op":"add","row":5}]}"#,
680 r#"{"epoch":1,"seq":0,"cv":0,"normalizedFp":"0","ops":[{"op":"add","row":[]}]}"#,
682 ];
683 for c in cases {
684 let v: Value = serde_json::from_str(c).unwrap();
685 assert!(
686 normalized_batch_from_json(&v).is_err(),
687 "should reject: {c}"
688 );
689 }
690
691 let hellos = [
692 r#"{"epoch":1,"comparatorVersion":1,"tables":[{"name":"t","columns":[1],"primaryKey":[0]}],"normalizedFp":"0"}"#,
694 r#"{"epoch":1,"comparatorVersion":1,"tables":[{"name":"t","columns":["a"],"primaryKey":[0.5]}],"normalizedFp":"0"}"#,
696 r#"{"epoch":1,"comparatorVersion":4294967296,"tables":[],"normalizedFp":"0"}"#,
698 r#"{"epoch":1,"comparatorVersion":1,"tables":{},"normalizedFp":"0"}"#,
700 ];
701 for c in hellos {
702 let v: Value = serde_json::from_str(c).unwrap();
703 assert!(
704 normalized_hello_from_json(&v).is_err(),
705 "should reject: {c}"
706 );
707 }
708 }
709}