Add gossip replication between nodes
Nodes run anti-entropy over a TCP link: each advertises its version vector, the peer replies with the log entries it lacks, and ingest is idempotent so re-delivery and reconnect after a split both converge. The engine is shared behind a mutex; config gains [gossip] and [[peer]].
This commit is contained in:
parent
82e41e95b2
commit
d0556ebe8c
7 changed files with 319 additions and 20 deletions
|
|
@ -65,26 +65,31 @@ pub struct EventLog {
|
|||
origin: String,
|
||||
lamport: u64, // logical clock, ticked on every event
|
||||
versions: HashMap<String, u64>, // per-origin highest seq applied (version vector)
|
||||
entries: Vec<LogEntry>, // full log, kept so peers can pull what they lack
|
||||
}
|
||||
|
||||
impl EventLog {
|
||||
fn open(path: PathBuf, origin: String) -> (Self, Vec<Event>) {
|
||||
let mut events = Vec::new();
|
||||
let mut lamport = 0;
|
||||
let mut versions: HashMap<String, u64> = HashMap::new();
|
||||
if let Ok(data) = std::fs::read_to_string(&path) {
|
||||
let mut log = Self { path, origin, lamport: 0, versions: HashMap::new(), entries: Vec::new() };
|
||||
if let Ok(data) = std::fs::read_to_string(&log.path) {
|
||||
for line in data.lines().filter(|l| !l.trim().is_empty()) {
|
||||
match serde_json::from_str::<LogEntry>(line) {
|
||||
Ok(entry) => {
|
||||
lamport = lamport.max(entry.lamport);
|
||||
versions.entry(entry.origin.clone()).and_modify(|s| *s = (*s).max(entry.seq)).or_insert(entry.seq);
|
||||
events.push(entry.event);
|
||||
log.absorb(&entry);
|
||||
log.entries.push(entry);
|
||||
}
|
||||
Err(e) => tracing::warn!(%e, "skipping malformed event log line"),
|
||||
}
|
||||
}
|
||||
}
|
||||
(Self { path, origin, lamport, versions }, events)
|
||||
let events = log.entries.iter().map(|e| e.event.clone()).collect();
|
||||
(log, events)
|
||||
}
|
||||
|
||||
// Roll the clock and version vector forward over an entry.
|
||||
fn absorb(&mut self, entry: &LogEntry) {
|
||||
self.lamport = self.lamport.max(entry.lamport);
|
||||
self.versions.entry(entry.origin.clone()).and_modify(|s| *s = (*s).max(entry.seq)).or_insert(entry.seq);
|
||||
}
|
||||
|
||||
// Seq the next locally-authored event will carry (0-based, per our origin).
|
||||
|
|
@ -99,13 +104,13 @@ impl EventLog {
|
|||
let entry = LogEntry { origin: self.origin.clone(), seq: self.next_seq(), lamport: self.lamport, event };
|
||||
self.persist(&entry)?;
|
||||
self.versions.insert(entry.origin.clone(), entry.seq);
|
||||
self.entries.push(entry);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ingest an entry authored by another node — the gossip seam. Returns the
|
||||
// event to fold into state, or None if already applied (idempotent, so
|
||||
// re-delivery converges). Assumes per-origin in-order delivery.
|
||||
#[allow(dead_code)]
|
||||
fn ingest(&mut self, entry: LogEntry) -> std::io::Result<Option<Event>> {
|
||||
if self.versions.get(&entry.origin).is_some_and(|&s| entry.seq <= s) {
|
||||
return Ok(None); // already have it
|
||||
|
|
@ -113,7 +118,23 @@ impl EventLog {
|
|||
self.persist(&entry)?;
|
||||
self.lamport = self.lamport.max(entry.lamport) + 1; // Lamport receive rule
|
||||
self.versions.insert(entry.origin.clone(), entry.seq);
|
||||
Ok(Some(entry.event))
|
||||
let event = entry.event.clone();
|
||||
self.entries.push(entry);
|
||||
Ok(Some(event))
|
||||
}
|
||||
|
||||
// Our version vector: highest seq applied per origin.
|
||||
fn version_vector(&self) -> HashMap<String, u64> {
|
||||
self.versions.clone()
|
||||
}
|
||||
|
||||
// Entries a peer is missing, given the version vector it advertised.
|
||||
fn missing_for(&self, peer: &HashMap<String, u64>) -> Vec<LogEntry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|e| peer.get(&e.origin).map_or(true, |&s| e.seq > s))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn persist(&self, entry: &LogEntry) -> std::io::Result<()> {
|
||||
|
|
@ -180,7 +201,6 @@ impl Db {
|
|||
|
||||
/// Fold an entry authored by another node into the store — the services-side
|
||||
/// of the gossip seam. Idempotent (re-delivered entries are dropped).
|
||||
#[allow(dead_code)]
|
||||
pub fn ingest(&mut self, entry: LogEntry) -> std::io::Result<()> {
|
||||
if let Some(event) = self.log.ingest(entry)? {
|
||||
apply(&mut self.accounts, event);
|
||||
|
|
@ -188,6 +208,16 @@ impl Db {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Our version vector, advertised to peers so they can send what we lack.
|
||||
pub fn version_vector(&self) -> HashMap<String, u64> {
|
||||
self.log.version_vector()
|
||||
}
|
||||
|
||||
/// The log entries a peer is missing, given the version vector it sent.
|
||||
pub fn missing_for(&self, peer: &HashMap<String, u64>) -> Vec<LogEntry> {
|
||||
self.log.missing_for(peer)
|
||||
}
|
||||
|
||||
pub fn exists(&self, name: &str) -> bool {
|
||||
self.accounts.contains_key(&key(name))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use std::time::{Duration, Instant};
|
|||
use base64::{engine::general_purpose::STANDARD, Engine as _};
|
||||
|
||||
use crate::proto::{NetAction, NetEvent, RegReply};
|
||||
use db::{Db, RegError};
|
||||
use db::{Db, LogEntry, RegError};
|
||||
use scram::Verifier;
|
||||
use service::{Sender, Service, ServiceCtx};
|
||||
use state::Network;
|
||||
|
|
@ -91,6 +91,29 @@ impl Engine {
|
|||
self.db.scram_iterations
|
||||
}
|
||||
|
||||
// Gossip pass-throughs to the account store, used by the replication layer.
|
||||
pub fn gossip_digest(&self) -> HashMap<String, u64> {
|
||||
self.db.version_vector()
|
||||
}
|
||||
|
||||
pub fn gossip_missing(&self, peer: &HashMap<String, u64>) -> Vec<LogEntry> {
|
||||
self.db.missing_for(peer)
|
||||
}
|
||||
|
||||
pub fn gossip_ingest(&mut self, entry: LogEntry) -> std::io::Result<()> {
|
||||
self.db.ingest(entry)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_register(&mut self, name: &str) {
|
||||
self.db.register(name, "pw", None).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_has_account(&self, name: &str) -> bool {
|
||||
self.db.exists(name)
|
||||
}
|
||||
|
||||
// Insert or refresh a client's in-progress SASL session, stamped now.
|
||||
fn stash_sasl(&mut self, client: String, session: SaslSession) {
|
||||
self.sasl_sessions.insert(client, TimedSession { touched: Instant::now(), session });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue