socketengine: run direct TLS in the reactor pool — non-blocking handshake + crypto in the worker threads (Sock::Tls, TlsSession), unifying the client I/O model and spreading TLS work across cores; proxied TLS + links keep the thread path

src/tls.rs

@@ -9,8 +9,11 @@ use std::io::{self, Read, Write};
 use std::net::{Shutdown, TcpStream};
 use std::time::Duration;
 
+use mio::net::TcpStream as MioStream;
 use openssl::hash::MessageDigest;
-use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod, SslStream, SslVerifyMode};
+use openssl::ssl::{
+    ErrorCode, Ssl, SslAcceptor, SslFiletype, SslMethod, SslMode, SslStream, SslVerifyMode,
+};
 
 /// A live TLS connection: read/write plaintext, tune the read timeout (the
 /// socket engine polls with one to interleave reads and queued writes), and shut
@@ -26,9 +29,31 @@ pub trait TlsConn: Send {
     fn peer_cert_fp(&self) -> Option<String>;
 }
 
-/// A TLS backend: performs the server-side handshake on an accepted socket.
+/// A non-blocking TLS session the reactor drives itself over a mio socket. The
+/// handshake and all reads/writes surface `WouldBlock` (mapped from OpenSSL's
+/// WANT_READ/WANT_WRITE) so the worker can register interest and come back later
+/// instead of blocking a whole thread on one connection.
+pub trait TlsSession: Send {
+    /// Drive the server handshake: `Ok(true)` once complete, `Ok(false)` while it
+    /// still needs I/O, `Err` on a fatal handshake failure.
+    fn accept(&mut self) -> io::Result<bool>;
+    /// Decrypt application data. `Ok(0)` means the peer sent a clean TLS close.
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize>;
+    /// Encrypt+queue application data; returns the plaintext bytes accepted.
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize>;
+    /// The underlying mio socket, for the reactor's poll (re)registration.
+    fn source(&mut self) -> &mut MioStream;
+    /// SHA-256 fingerprint of the peer certificate (CertFP / SASL EXTERNAL), if any.
+    fn peer_cert_fp(&self) -> Option<String>;
+    fn shutdown(&mut self);
+}
+
+/// A TLS backend: wraps an accepted socket in a TLS session — either blocking
+/// ([`accept`], the thread-per-connection path) or non-blocking ([`start`], the
+/// reactor path).
 pub trait TlsBackend: Send + Sync {
     fn accept(&self, sock: TcpStream) -> io::Result<Box<dyn TlsConn>>;
+    fn start(&self, sock: MioStream) -> io::Result<Box<dyn TlsSession>>;
 }
 
 fn err<E: std::fmt::Display>(e: E) -> io::Error {
@@ -52,6 +77,10 @@ impl OpensslBackend {
         // read its fingerprint. We never validate the chain — services match the
         // fingerprint to an account — so the callback always accepts.
         b.set_verify_callback(SslVerifyMode::PEER, |_valid, _ctx| true);
+        // The reactor drives writes non-blocking and may retry SSL_write with a moved
+        // or grown buffer after a WouldBlock; allow that and partial progress so a slow
+        // TLS reader can't wedge a worker.
+        b.set_mode(SslMode::ENABLE_PARTIAL_WRITE | SslMode::ACCEPT_MOVING_WRITE_BUFFER);
         Ok(OpensslBackend {
             acceptor: b.build(),
         })
@@ -63,6 +92,59 @@ impl TlsBackend for OpensslBackend {
         let stream = self.acceptor.accept(sock).map_err(err)?;
         Ok(Box::new(OpensslConn(stream)))
     }
+
+    fn start(&self, sock: MioStream) -> io::Result<Box<dyn TlsSession>> {
+        let ssl = Ssl::new(self.acceptor.context()).map_err(err)?;
+        // handshake isn't driven here: SslStream::new just binds the socket; the
+        // reactor calls accept() as the socket becomes readable/writable.
+        let stream = SslStream::new(ssl, sock).map_err(err)?;
+        Ok(Box::new(OpensslSession(stream)))
+    }
+}
+
+struct OpensslSession(SslStream<MioStream>);
+
+/// Map an OpenSSL ssl error to the reactor's io model: WANT_READ/WANT_WRITE ⇒
+/// `WouldBlock` (retry when ready), everything else ⇒ a real error.
+fn ssl_io_err(e: openssl::ssl::Error) -> io::Error {
+    match e.code() {
+        ErrorCode::WANT_READ | ErrorCode::WANT_WRITE => io::ErrorKind::WouldBlock.into(),
+        _ => e.into_io_error().unwrap_or_else(io::Error::other),
+    }
+}
+
+impl TlsSession for OpensslSession {
+    fn accept(&mut self) -> io::Result<bool> {
+        match self.0.accept() {
+            Ok(()) => Ok(true),
+            Err(e) => match e.code() {
+                ErrorCode::WANT_READ | ErrorCode::WANT_WRITE => Ok(false),
+                _ => Err(e.into_io_error().unwrap_or_else(io::Error::other)),
+            },
+        }
+    }
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self.0.ssl_read(buf) {
+            Ok(n) => Ok(n),
+            // a clean TLS close is EOF, like a plaintext socket returning 0
+            Err(e) if e.code() == ErrorCode::ZERO_RETURN => Ok(0),
+            Err(e) => Err(ssl_io_err(e)),
+        }
+    }
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        self.0.ssl_write(buf).map_err(ssl_io_err)
+    }
+    fn source(&mut self) -> &mut MioStream {
+        self.0.get_mut()
+    }
+    fn peer_cert_fp(&self) -> Option<String> {
+        let cert = self.0.ssl().peer_certificate()?;
+        let digest = cert.digest(MessageDigest::sha256()).ok()?;
+        Some(digest.iter().map(|b| format!("{b:02x}")).collect())
+    }
+    fn shutdown(&mut self) {
+        let _ = self.0.get_ref().shutdown(Shutdown::Both);
+    }
 }
 
 struct OpensslConn(SslStream<TcpStream>);