diff --git a/docs/federation.md b/docs/federation.md index b652244..f05a670 100644 --- a/docs/federation.md +++ b/docs/federation.md @@ -23,11 +23,16 @@ Each node owns an append-only event log. The log is split into two scopes: Nodes sync via anti-entropy: each advertises a per-origin version vector, and peers send whatever Global entries the other is missing. Delivery is idempotent, -so a re-sent entry is dropped. Entries are applied strictly **in order per -origin**; one that arrives ahead of its predecessors (a relay in a 3+-node mesh -racing the direct path) is not applied out of order — the version vector stays at -the last contiguous seq, so the digest re-requests the gap in sequence rather -than losing it. +so a re-sent entry is dropped, and each account/channel event is an idempotent +upsert. A peer converges by accepting any entry newer than its version vector +(including the forward jump a compacted node's log begins at). + +> **Known limitation (3+ nodes).** In a relay mesh, an entry can reach a node +> ahead of its predecessors via a longer path, and the current ingest advances the +> version vector past that gap. With two nodes (direct delivery) this never +> happens. Closing it for larger meshes needs a snapshot-epoch marker so a +> compaction jump is distinguishable from a live gap; until then, prefer 2-node +> topologies. This is dormant today (no peers). ## The trust model @@ -75,8 +80,9 @@ Rules: all account data are on that wire. Echo warns loudly at startup if TLS is off. - **Firewall** the `bind` port to your peer nodes' IPs only. - Use a strong random `secret` and rotate it periodically. -- The 2-node path is the most exercised. 3+ nodes work (out-of-order delivery is - buffered), but test your topology before relying on it. +- **Prefer 2 nodes.** Direct delivery keeps replication in order; a 3+-node relay + mesh has a known out-of-order edge (see "Known limitation" above) that isn't + closed yet. ## Tier C — cross-operator federation (per-origin signing) diff --git a/src/engine/db/mod.rs b/src/engine/db/mod.rs index 341aa7e..7a6b156 100644 --- a/src/engine/db/mod.rs +++ b/src/engine/db/mod.rs @@ -894,20 +894,16 @@ impl EventLog { return Ok(None); } } - // Apply strictly in per-origin sequence. The FIRST entry ever seen from an - // origin sets the baseline — a peer syncing from a COMPACTED node starts - // mid-stream (its seqs restart at next_seq(), not 0), so we can't demand 0. - // After that, only the next contiguous seq is accepted; a non-contiguous one - // (an older duplicate, or a future entry a relay delivered ahead of the direct - // path in a 3+-node mesh) is dropped rather than applied, so it can't advance - // the version vector past a gap — the digest then re-requests the gap in order - // instead of losing it forever. - let expected = match self.versions.get(&entry.origin) { - None => entry.seq, - Some(&s) => s + 1, - }; - if entry.seq != expected { - return Ok(None); + // Accept any entry newer than what we hold for this origin. A forward jump is + // legitimate here: a peer syncing from a COMPACTED node receives seqs that + // start well above 0 (the intermediate ones were folded into the snapshot), + // and the fold is an idempotent upsert. NOTE: this cannot detect an out-of- + // order relay in a 3+-node mesh (an entry delivered ahead of its predecessors + // advances the version past the gap). Fixing that safely needs a snapshot-epoch + // marker so a compaction jump is distinguishable from a live gap — deferred; + // see docs/federation.md. Not reachable today (no gossip peers). + if self.versions.get(&entry.origin).is_some_and(|&s| entry.seq <= s) { + return Ok(None); // already have it } self.persist(&entry)?; self.lamport = self.lamport.max(entry.lamport).saturating_add(1); // Lamport receive rule diff --git a/src/engine/tests.rs b/src/engine/tests.rs index f226ecb..639a105 100644 --- a/src/engine/tests.rs +++ b/src/engine/tests.rs @@ -878,34 +878,6 @@ assert!(lifted, "gossiped ban lift removes the line from the ircd"); } - // A gossiped entry that arrives out of sequence (a relay racing the direct path - // in a 3+-node mesh) must NOT advance the version vector past the gap, or the - // skipped entries are lost forever; the digest re-requests them in order. - #[test] - fn gossip_drops_an_out_of_order_entry_until_the_gap_fills() { - use echo_nickserv::NickServ; - let path = std::env::temp_dir().join("echo-gossip-ooo.jsonl"); - let _ = std::fs::remove_file(&path); - let mut e = Engine::new(vec![Box::new(NickServ { uid: "42SAAAAAA".into(), guest_nick: "Guest".into(), guest_seq: 0 })], Db::open(&path, "test")); - e.set_sid("42S".into()); - let (tx, _rx) = tokio::sync::mpsc::unbounded_channel(); - e.set_irc_out(tx); - let akill = |seq: u64, mask: &str| LogEntry::for_test("peer", seq, seq + 1, db::Event::AkillAdded { - kind: "G".into(), mask: mask.into(), setter: "op".into(), reason: "x".into(), ts: 0, expires: None, - }); - - e.gossip_ingest(akill(0, "*@a")).unwrap(); - assert_eq!(e.gossip_digest().get("peer"), Some(&0), "first entry applies"); - // Seq 2 arrives before seq 1: dropped, version stays at 0. - e.gossip_ingest(akill(2, "*@c")).unwrap(); - assert_eq!(e.gossip_digest().get("peer"), Some(&0), "an out-of-order entry doesn't advance the version"); - // Seq 1 fills the gap, then the re-delivered seq 2 is contiguous. - e.gossip_ingest(akill(1, "*@b")).unwrap(); - assert_eq!(e.gossip_digest().get("peer"), Some(&1)); - e.gossip_ingest(akill(2, "*@c")).unwrap(); - assert_eq!(e.gossip_digest().get("peer"), Some(&2), "the gap filled, seq 2 now applies"); - } - // Losing a registration conflict logs out the local session on that name and // notifies them over the services-initiated outbound path. #[test]