Add gossip replication between nodes

Nodes run anti-entropy over a TCP link: each advertises its version
vector, the peer replies with the log entries it lacks, and ingest is
idempotent so re-delivery and reconnect after a split both converge.
The engine is shared behind a mutex; config gains [gossip] and [[peer]].
This commit is contained in:
Jean Chevronnet 2026-07-12 07:05:07 +00:00
parent 82e41e95b2
commit d0556ebe8c
No known key found for this signature in database
7 changed files with 319 additions and 20 deletions

View file

@ -65,26 +65,31 @@ pub struct EventLog {
origin: String,
lamport: u64, // logical clock, ticked on every event
versions: HashMap<String, u64>, // per-origin highest seq applied (version vector)
entries: Vec<LogEntry>, // full log, kept so peers can pull what they lack
}
impl EventLog {
fn open(path: PathBuf, origin: String) -> (Self, Vec<Event>) {
let mut events = Vec::new();
let mut lamport = 0;
let mut versions: HashMap<String, u64> = HashMap::new();
if let Ok(data) = std::fs::read_to_string(&path) {
let mut log = Self { path, origin, lamport: 0, versions: HashMap::new(), entries: Vec::new() };
if let Ok(data) = std::fs::read_to_string(&log.path) {
for line in data.lines().filter(|l| !l.trim().is_empty()) {
match serde_json::from_str::<LogEntry>(line) {
Ok(entry) => {
lamport = lamport.max(entry.lamport);
versions.entry(entry.origin.clone()).and_modify(|s| *s = (*s).max(entry.seq)).or_insert(entry.seq);
events.push(entry.event);
log.absorb(&entry);
log.entries.push(entry);
}
Err(e) => tracing::warn!(%e, "skipping malformed event log line"),
}
}
}
(Self { path, origin, lamport, versions }, events)
let events = log.entries.iter().map(|e| e.event.clone()).collect();
(log, events)
}
// Roll the clock and version vector forward over an entry.
fn absorb(&mut self, entry: &LogEntry) {
self.lamport = self.lamport.max(entry.lamport);
self.versions.entry(entry.origin.clone()).and_modify(|s| *s = (*s).max(entry.seq)).or_insert(entry.seq);
}
// Seq the next locally-authored event will carry (0-based, per our origin).
@ -99,13 +104,13 @@ impl EventLog {
let entry = LogEntry { origin: self.origin.clone(), seq: self.next_seq(), lamport: self.lamport, event };
self.persist(&entry)?;
self.versions.insert(entry.origin.clone(), entry.seq);
self.entries.push(entry);
Ok(())
}
// Ingest an entry authored by another node — the gossip seam. Returns the
// event to fold into state, or None if already applied (idempotent, so
// re-delivery converges). Assumes per-origin in-order delivery.
#[allow(dead_code)]
fn ingest(&mut self, entry: LogEntry) -> std::io::Result<Option<Event>> {
if self.versions.get(&entry.origin).is_some_and(|&s| entry.seq <= s) {
return Ok(None); // already have it
@ -113,7 +118,23 @@ impl EventLog {
self.persist(&entry)?;
self.lamport = self.lamport.max(entry.lamport) + 1; // Lamport receive rule
self.versions.insert(entry.origin.clone(), entry.seq);
Ok(Some(entry.event))
let event = entry.event.clone();
self.entries.push(entry);
Ok(Some(event))
}
// Our version vector: highest seq applied per origin.
fn version_vector(&self) -> HashMap<String, u64> {
self.versions.clone()
}
// Entries a peer is missing, given the version vector it advertised.
fn missing_for(&self, peer: &HashMap<String, u64>) -> Vec<LogEntry> {
self.entries
.iter()
.filter(|e| peer.get(&e.origin).map_or(true, |&s| e.seq > s))
.cloned()
.collect()
}
fn persist(&self, entry: &LogEntry) -> std::io::Result<()> {
@ -180,7 +201,6 @@ impl Db {
/// Fold an entry authored by another node into the store — the services-side
/// of the gossip seam. Idempotent (re-delivered entries are dropped).
#[allow(dead_code)]
pub fn ingest(&mut self, entry: LogEntry) -> std::io::Result<()> {
if let Some(event) = self.log.ingest(entry)? {
apply(&mut self.accounts, event);
@ -188,6 +208,16 @@ impl Db {
Ok(())
}
/// Our version vector, advertised to peers so they can send what we lack.
pub fn version_vector(&self) -> HashMap<String, u64> {
self.log.version_vector()
}
/// The log entries a peer is missing, given the version vector it sent.
pub fn missing_for(&self, peer: &HashMap<String, u64>) -> Vec<LogEntry> {
self.log.missing_for(peer)
}
pub fn exists(&self, name: &str) -> bool {
self.accounts.contains_key(&key(name))
}

View file

@ -9,7 +9,7 @@ use std::time::{Duration, Instant};
use base64::{engine::general_purpose::STANDARD, Engine as _};
use crate::proto::{NetAction, NetEvent, RegReply};
use db::{Db, RegError};
use db::{Db, LogEntry, RegError};
use scram::Verifier;
use service::{Sender, Service, ServiceCtx};
use state::Network;
@ -91,6 +91,29 @@ impl Engine {
self.db.scram_iterations
}
// Gossip pass-throughs to the account store, used by the replication layer.
pub fn gossip_digest(&self) -> HashMap<String, u64> {
self.db.version_vector()
}
pub fn gossip_missing(&self, peer: &HashMap<String, u64>) -> Vec<LogEntry> {
self.db.missing_for(peer)
}
pub fn gossip_ingest(&mut self, entry: LogEntry) -> std::io::Result<()> {
self.db.ingest(entry)
}
#[cfg(test)]
pub(crate) fn test_register(&mut self, name: &str) {
self.db.register(name, "pw", None).unwrap();
}
#[cfg(test)]
pub(crate) fn test_has_account(&self, name: &str) -> bool {
self.db.exists(name)
}
// Insert or refresh a client's in-progress SASL session, stamped now.
fn stash_sasl(&mut self, client: String, session: SaslSession) {
self.sasl_sessions.insert(client, TimedSession { touched: Instant::now(), session });