fedserv/echo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Push new log entries to peers on commit

Browse source 
A broadcast channel forwards each freshly committed entry straight to
connected peers, so a registration propagates in milliseconds. The
periodic digest drops to a 10s anti-entropy backstop.
This commit is contained in:
Jean Chevronnet 2026-07-12 07:20:25 +00:00
parent d0556ebe8c
commit df7b413862
No known key found for this signature in database
3 changed files with 101 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

99
src/gossip.rs
View file

@ -8,15 +8,16 @@ use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{mpsc, Mutex};
use tokio::sync::{broadcast, mpsc, Mutex};
use crate::config::{Gossip, Peer};
use crate::engine::db::LogEntry;
use crate::engine::Engine;
type Shared = Arc<Mutex<Engine>>;
type Outbound = broadcast::Sender<LogEntry>;
const TICK: Duration = Duration::from_secs(3); // how often we re-advertise our vector
const TICK: Duration = Duration::from_secs(10); // anti-entropy backstop; pushes handle the common case
const REDIAL: Duration = Duration::from_secs(5); // reconnect backoff for dialed peers
// Wire protocol: one JSON object per line.
@ -29,16 +30,16 @@ enum Msg {
}
// Start the listener (if bound) and a dialer per configured peer.
pub async fn run(engine: Shared, cfg: Gossip, peers: Vec<Peer>, origin: String) {
pub async fn run(engine: Shared, cfg: Gossip, peers: Vec<Peer>, origin: String, outbound: Outbound) {
if let Some(bind) = cfg.bind.clone() {
tokio::spawn(listen(bind, engine.clone(), cfg.secret.clone(), origin.clone()));
tokio::spawn(listen(bind, engine.clone(), cfg.secret.clone(), origin.clone(), outbound.clone()));
}
for peer in peers {
tokio::spawn(dial(peer.addr, engine.clone(), cfg.secret.clone(), origin.clone()));
tokio::spawn(dial(peer.addr, engine.clone(), cfg.secret.clone(), origin.clone(), outbound.clone()));
}
}
async fn listen(bind: String, engine: Shared, secret: String, origin: String) {
async fn listen(bind: String, engine: Shared, secret: String, origin: String, outbound: Outbound) {
let listener = match TcpListener::bind(&bind).await {
Ok(l) => l,
Err(e) => return tracing::error!(%e, %bind, "gossip bind failed"),
@ -47,9 +48,9 @@ async fn listen(bind: String, engine: Shared, secret: String, origin: String) {
loop {
if let Ok((stream, addr)) = listener.accept().await {
tracing::info!(%addr, "gossip peer accepted");
let (engine, secret, origin) = (engine.clone(), secret.clone(), origin.clone());
let (engine, secret, origin, outbound) = (engine.clone(), secret.clone(), origin.clone(), outbound.clone());
tokio::spawn(async move {
if let Err(e) = session(stream, engine, secret, origin).await {
if let Err(e) = session(stream, engine, secret, origin, outbound).await {
tracing::debug!(%e, "gossip session ended");
}
});
@ -57,12 +58,12 @@ async fn listen(bind: String, engine: Shared, secret: String, origin: String) {
}
}
async fn dial(addr: String, engine: Shared, secret: String, origin: String) {
async fn dial(addr: String, engine: Shared, secret: String, origin: String, outbound: Outbound) {
loop {
match TcpStream::connect(&addr).await {
Ok(stream) => {
tracing::info!(%addr, "gossip dialed peer");
if let Err(e) = session(stream, engine.clone(), secret.clone(), origin.clone()).await {
if let Err(e) = session(stream, engine.clone(), secret.clone(), origin.clone(), outbound.clone()).await {
tracing::debug!(%e, %addr, "gossip session ended");
}
}
@ -73,7 +74,7 @@ async fn dial(addr: String, engine: Shared, secret: String, origin: String) {
}
// One peer connection: authenticate, then run anti-entropy until it drops.
async fn session<S>(stream: S, engine: Shared, secret: String, origin: String) -> anyhow::Result<()>
async fn session<S>(stream: S, engine: Shared, secret: String, origin: String, outbound: Outbound) -> anyhow::Result<()>
where
S: AsyncRead + AsyncWrite + Send + 'static,
{
@ -106,7 +107,9 @@ where
}
}
// Re-advertise our version vector on a timer so new local events propagate.
// Re-advertise our version vector on a timer. The first tick fires an
// immediate digest for initial sync; later ones are a backstop for anything
// a push dropped.
let ticker = {
let (tx, engine) = (tx.clone(), engine.clone());
tokio::spawn(async move {
@ -120,6 +123,26 @@ where
})
};
// Push freshly committed entries straight to the peer, so a new registration
// propagates in milliseconds instead of waiting for the next digest. Idempotent
// ingest on the far side absorbs the echo of an entry the peer just sent us.
let forwarder = {
let (tx, mut rx) = (tx.clone(), outbound.subscribe());
tokio::spawn(async move {
loop {
match rx.recv().await {
Ok(entry) => {
if send(&tx, &Msg::Entry { entry }).await.is_err() {
break;
}
}
Err(broadcast::error::RecvError::Lagged(_)) => continue, // digest backstop catches up
Err(broadcast::error::RecvError::Closed) => break,
}
}
})
};
// Answer a peer's digest with what it lacks; apply the entries it sends us.
let result = loop {
match reader.next_line().await {
@ -143,6 +166,7 @@ where
};
ticker.abort();
forwarder.abort();
writer.abort();
result
}
@ -158,25 +182,28 @@ mod tests {
use crate::engine::db::Db;
use crate::services::nickserv::NickServ;
fn engine(origin: &str, tag: &str) -> Shared {
fn engine(origin: &str, tag: &str) -> (Shared, Outbound) {
let path = std::env::temp_dir().join(format!("fedserv-gossip-{tag}.jsonl"));
let _ = std::fs::remove_file(&path);
let (tx, _) = broadcast::channel(1024);
let mut db = Db::open(&path, origin);
db.scram_iterations = 4096;
db.set_outbound(tx.clone());
let ns = NickServ { uid: format!("{origin}AAAAAA"), guest_nick: "Guest".into(), guest_seq: 0 };
Arc::new(Mutex::new(Engine::new(vec![Box::new(ns)], db)))
(Arc::new(Mutex::new(Engine::new(vec![Box::new(ns)], db))), tx)
}
// An account registered on node A reaches node B over a gossip link.
// An account that exists before the link comes up reaches the peer via the
// initial digest exchange (the pull path).
#[tokio::test]
async fn two_nodes_converge_over_a_link() {
let a = engine("A", "conv-a");
let b = engine("B", "conv-b");
let (a, atx) = engine("A", "conv-a");
let (b, btx) = engine("B", "conv-b");
a.lock().await.test_register("alice");
let (ca, cb) = tokio::io::duplex(64 * 1024);
let sa = tokio::spawn(session(ca, a.clone(), "s3cret".into(), "A".into()));
let sb = tokio::spawn(session(cb, b.clone(), "s3cret".into(), "B".into()));
let sa = tokio::spawn(session(ca, a.clone(), "s3cret".into(), "A".into(), atx));
let sb = tokio::spawn(session(cb, b.clone(), "s3cret".into(), "B".into(), btx));
let mut converged = false;
for _ in 0..100 {
@ -191,16 +218,42 @@ mod tests {
assert!(converged, "B should receive A's account over gossip");
}
// A write made after the link is up must arrive within the poll interval,
// which only the push path can do (the digest backstop is 10s).
#[tokio::test]
async fn push_propagates_new_writes() {
let (a, atx) = engine("A", "push-a");
let (b, btx) = engine("B", "push-b");
let (ca, cb) = tokio::io::duplex(64 * 1024);
let sa = tokio::spawn(session(ca, a.clone(), "s3cret".into(), "A".into(), atx));
let sb = tokio::spawn(session(cb, b.clone(), "s3cret".into(), "B".into(), btx));
tokio::time::sleep(Duration::from_millis(200)).await; // let both subscribe
a.lock().await.test_register("late");
let mut got = false;
for _ in 0..40 {
if b.lock().await.test_has_account("late") {
got = true;
break;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
sa.abort();
sb.abort();
assert!(got, "a post-connect write should push to B well under the 10s digest");
}
// A wrong secret is rejected before any state is exchanged.
#[tokio::test]
async fn bad_secret_is_rejected() {
let a = engine("A", "auth-a");
let b = engine("B", "auth-b");
let (a, atx) = engine("A", "auth-a");
let (b, btx) = engine("B", "auth-b");
a.lock().await.test_register("alice");
let (ca, cb) = tokio::io::duplex(64 * 1024);
let sa = tokio::spawn(session(ca, a.clone(), "right".into(), "A".into()));
let sb = tokio::spawn(session(cb, b.clone(), "wrong".into(), "B".into()));
let sa = tokio::spawn(session(ca, a.clone(), "right".into(), "A".into(), atx));
let sb = tokio::spawn(session(cb, b.clone(), "wrong".into(), "B".into(), btx));
let _ = tokio::time::timeout(Duration::from_secs(1), async {
let _ = sa.await;