resilience: isolate per-connection panics in the plaintext reactor (catch_unwind read/write -> drop just that conn) and log worker-thread panics instead of vanishing silently

This commit is contained in:
Jean Chevronnet 2026-08-12 13:19:25 +00:00
parent 145a01b2c2
commit e91b64a4db
2 changed files with 24 additions and 4 deletions

View file

@ -479,8 +479,14 @@ impl Server {
let tx = self.event_tx.clone(); let tx = self.event_tx.clone();
std::thread::spawn(move || { std::thread::spawn(move || {
let _guard = Guard; // decrements even on panic let _guard = Guard; // decrements even on panic
let ev = f(); match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
Ok(ev) => {
let _ = tx.send(ev); let _ = tx.send(ev);
}
// a panic here can't reach the core (we don't know which Event to send);
// log it so a stuck request is diagnosable instead of silent.
Err(_) => eprintln!("[worker] a background crypto/http task panicked; its request was dropped"),
}
}); });
true true
} }

View file

@ -351,11 +351,25 @@ pub fn run_reactor(
} }
} }
Token(t) => { Token(t) => {
// isolate per-connection I/O: a panic framing one client's bytes
// drops that client, never the reactor that serves all the others.
if event.is_readable() { if event.is_readable() {
read_conn(&mut poll, &mut conns, t, &core); let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
read_conn(&mut poll, &mut conns, t, &core)
}));
if r.is_err() {
eprintln!("[reactor] recovered from a panic reading a socket; dropping that connection");
close_conn(&mut poll, &mut conns, t, &core);
}
} }
if event.is_writable() && conns.contains_key(&t) { if event.is_writable() && conns.contains_key(&t) {
flush_conn(&mut poll, &mut conns, t, &core); let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
flush_conn(&mut poll, &mut conns, t, &core)
}));
if r.is_err() {
eprintln!("[reactor] recovered from a panic writing a socket; dropping that connection");
close_conn(&mut poll, &mut conns, t, &core);
}
} }
} }
} }