Also fixes two pre-existing test call sites that NAP-328 left passing the old arity to register()/state_for() after adding home_dir.
299 lines
11 KiB
Rust
299 lines
11 KiB
Rust
//! Per-connection handler for the `WS /daemon` endpoint.
|
|
//!
|
|
//! The handler is split from the axum-specific glue so it can be exercised
|
|
//! directly with an in-process pair of channels in tests. The axum side just
|
|
//! adapts a `WebSocket` into the trait below.
|
|
|
|
use std::time::Duration;
|
|
|
|
use axum::extract::ws::{Message, WebSocket};
|
|
use futures_util::{SinkExt, StreamExt};
|
|
use maestro_protocol::{DaemonInbound, DaemonOutbound, ErrorInfo, RegistryEvent};
|
|
use tokio::sync::broadcast::error::RecvError;
|
|
use tokio::sync::mpsc;
|
|
|
|
use crate::state::Registry;
|
|
|
|
/// Bound the registry's per-daemon outbound queue. 64 in-flight R→D requests
|
|
/// is plenty for a single daemon and bounds memory use under back-pressure.
|
|
const OUTBOUND_BUFFER: usize = 64;
|
|
|
|
/// Run the handshake + message loop on a freshly-accepted WebSocket. Returns
|
|
/// the `daemon_id` once the connection terminates (for logging).
|
|
///
|
|
/// `heartbeat_timeout` is the maximum gap allowed between inbound frames —
|
|
/// once exceeded, the connection is dropped and the daemon's sessions are
|
|
/// removed from the catalog. Production code passes
|
|
/// [`crate::state::HEARTBEAT_TIMEOUT`]; tests use a short value.
|
|
pub async fn handle_socket(
|
|
socket: WebSocket,
|
|
registry: Registry,
|
|
expected_token: &str,
|
|
heartbeat_timeout: Duration,
|
|
) -> Option<String> {
|
|
let (mut sink, mut stream) = socket.split();
|
|
|
|
let hello = match read_hello(&mut stream).await {
|
|
Some(h) => h,
|
|
None => return None,
|
|
};
|
|
|
|
let (daemon_id, hostname, version, token, daemon_url, home_dir) = match hello {
|
|
DaemonOutbound::Hello {
|
|
daemon_id,
|
|
hostname,
|
|
version,
|
|
token,
|
|
daemon_url,
|
|
home_dir,
|
|
} => (daemon_id, hostname, version, token, daemon_url, home_dir),
|
|
_ => {
|
|
let _ = send_error(&mut sink, "expected_hello", "first message must be Hello").await;
|
|
return None;
|
|
}
|
|
};
|
|
|
|
if token != expected_token {
|
|
let _ = send_error(&mut sink, "auth_failed", "invalid token").await;
|
|
return None;
|
|
}
|
|
|
|
let (out_tx, mut out_rx) = mpsc::channel::<DaemonInbound>(OUTBOUND_BUFFER);
|
|
// Hold the fencing token for the life of this connection; only a deregister
|
|
// carrying it may tear down the entry we just installed (NAP-258).
|
|
let connection_token = registry
|
|
.register(
|
|
daemon_id.clone(),
|
|
hostname,
|
|
version,
|
|
daemon_url,
|
|
home_dir,
|
|
out_tx.clone(),
|
|
)
|
|
.await;
|
|
|
|
let _ = sink
|
|
.send(Message::Text(
|
|
serde_json::to_string(&DaemonInbound::Ack).expect("ack serialize"),
|
|
))
|
|
.await;
|
|
|
|
// NAP-246: forward peer-filtered catalog deltas to the daemon. Initial
|
|
// snapshot is pushed eagerly; live deltas flow through `catalog_task` →
|
|
// `out_tx` so they share the existing outbound writer in the select! loop
|
|
// (any extra sink writer would contend with the WS sink+stream coupling
|
|
// axum imposes and stall the inbound reader — see history of this file).
|
|
let (snapshot, mut catalog_rx) = registry.subscribe().await;
|
|
if let Some(peer_snapshot) = snapshot.for_peer(&daemon_id) {
|
|
if out_tx
|
|
.send(DaemonInbound::CatalogEvent {
|
|
event: peer_snapshot,
|
|
})
|
|
.await
|
|
.is_err()
|
|
{
|
|
registry.deregister(&daemon_id, connection_token).await;
|
|
return Some(daemon_id);
|
|
}
|
|
}
|
|
let catalog_id = daemon_id.clone();
|
|
let catalog_out = out_tx.clone();
|
|
let catalog_task = tokio::spawn(async move {
|
|
loop {
|
|
let ev = match catalog_rx.recv().await {
|
|
Ok(e) => e.for_peer(&catalog_id),
|
|
Err(RecvError::Lagged(n)) => Some(RegistryEvent::Lagged { skipped: n }),
|
|
Err(RecvError::Closed) => return,
|
|
};
|
|
let Some(event) = ev else { continue };
|
|
if catalog_out
|
|
.send(DaemonInbound::CatalogEvent { event })
|
|
.await
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
});
|
|
|
|
loop {
|
|
tokio::select! {
|
|
inbound = tokio::time::timeout(heartbeat_timeout, stream.next()) => {
|
|
match inbound {
|
|
Err(_) => {
|
|
tracing::info!(%daemon_id, "heartbeat timeout — dropping");
|
|
break;
|
|
}
|
|
Ok(None) => break,
|
|
Ok(Some(Err(e))) => {
|
|
tracing::info!(%daemon_id, ?e, "ws read error");
|
|
break;
|
|
}
|
|
Ok(Some(Ok(Message::Close(_)))) => break,
|
|
Ok(Some(Ok(Message::Text(text)))) => {
|
|
if !dispatch_text(®istry, &daemon_id, expected_token, &text).await {
|
|
break;
|
|
}
|
|
}
|
|
Ok(Some(Ok(Message::Binary(_) | Message::Ping(_) | Message::Pong(_)))) => {}
|
|
}
|
|
}
|
|
Some(msg) = out_rx.recv() => {
|
|
let text = serde_json::to_string(&msg).expect("serialize daemon-inbound");
|
|
if sink.send(Message::Text(text)).await.is_err() {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
catalog_task.abort();
|
|
registry.deregister(&daemon_id, connection_token).await;
|
|
Some(daemon_id)
|
|
}
|
|
|
|
/// Reads the first text frame, parses it as a `DaemonOutbound`, returns it.
|
|
async fn read_hello(
|
|
stream: &mut futures_util::stream::SplitStream<WebSocket>,
|
|
) -> Option<DaemonOutbound> {
|
|
match tokio::time::timeout(Duration::from_secs(5), stream.next()).await {
|
|
Ok(Some(Ok(Message::Text(t)))) => serde_json::from_str(&t).ok(),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
async fn dispatch_text(
|
|
registry: &Registry,
|
|
daemon_id: &str,
|
|
shared_secret: &str,
|
|
text: &str,
|
|
) -> bool {
|
|
let parsed: DaemonOutbound = match serde_json::from_str(text) {
|
|
Ok(p) => p,
|
|
Err(e) => {
|
|
// A single malformed frame must not tear down the WS (and wipe this
|
|
// daemon's catalog) — log and skip it (NAP-269).
|
|
tracing::warn!(%daemon_id, error = %e, "malformed message — skipping");
|
|
return true;
|
|
}
|
|
};
|
|
match parsed {
|
|
DaemonOutbound::Hello { .. } => {
|
|
tracing::warn!(%daemon_id, "second Hello on live connection — ignoring");
|
|
}
|
|
DaemonOutbound::SessionsSnapshot { sessions } => {
|
|
registry.apply_snapshot(daemon_id, sessions).await;
|
|
}
|
|
DaemonOutbound::SessionAdded { session } => {
|
|
registry.add_session(daemon_id, session).await;
|
|
}
|
|
DaemonOutbound::SessionRemoved { id } => {
|
|
registry.remove_session(daemon_id, id).await;
|
|
}
|
|
DaemonOutbound::SessionUpdated { session } => {
|
|
registry.update_session(daemon_id, session).await;
|
|
}
|
|
DaemonOutbound::Heartbeat { perimeter } => {
|
|
// Per-read timeout above is the heartbeat watchdog; capture the
|
|
// perimeter advertisement so the broker can hand it back as a T2
|
|
// candidate (NAP-247).
|
|
registry.update_perimeter(daemon_id, perimeter).await;
|
|
}
|
|
DaemonOutbound::CreateSessionResult { reply_to, result } => {
|
|
registry.complete_create(reply_to, result).await;
|
|
}
|
|
DaemonOutbound::KillSessionResult { reply_to, result } => {
|
|
registry.complete_kill(reply_to, result).await;
|
|
}
|
|
DaemonOutbound::ResolveAttach {
|
|
reply_to,
|
|
session_id,
|
|
} => {
|
|
registry
|
|
.resolve_attach(daemon_id, reply_to, session_id, shared_secret)
|
|
.await;
|
|
}
|
|
DaemonOutbound::ForwardCreate { reply_to, request } => {
|
|
// Relay the create to its owning daemon and route the reply back to
|
|
// this (home) daemon — control-plane only (NAP-327).
|
|
registry.forward_create(daemon_id, reply_to, request).await;
|
|
}
|
|
DaemonOutbound::ForwardKill {
|
|
reply_to,
|
|
daemon_id: target,
|
|
id,
|
|
} => {
|
|
registry
|
|
.forward_kill(daemon_id, reply_to, &target, id)
|
|
.await;
|
|
}
|
|
DaemonOutbound::ListDirResult { reply_to, result } => {
|
|
registry.complete_list_dir(reply_to, result).await;
|
|
}
|
|
DaemonOutbound::ForwardListDir {
|
|
reply_to,
|
|
daemon_id: target,
|
|
path,
|
|
} => {
|
|
// Relay a directory listing to its owning daemon and route the reply
|
|
// back to this (home) daemon — control-plane only (NAP-329).
|
|
registry
|
|
.forward_list_dir(daemon_id, reply_to, &target, path)
|
|
.await;
|
|
}
|
|
DaemonOutbound::RelayAck { reply_to, .. } => {
|
|
// T3 (relay) is gated on NAP-238. Land the variant so newer daemons
|
|
// don't crash the connection; the registry doesn't act on it yet.
|
|
tracing::debug!(%daemon_id, ?reply_to, "RelayAck received but T3 not implemented (NAP-238 pending)");
|
|
}
|
|
DaemonOutbound::Unknown => {
|
|
// A newer daemon emitted a frame type this build doesn't know. Skip
|
|
// it rather than dropping the connection (NAP-269).
|
|
tracing::debug!(%daemon_id, "unknown frame type — skipping (forward-compat, NAP-269)");
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
async fn send_error(
|
|
sink: &mut futures_util::stream::SplitSink<WebSocket, Message>,
|
|
code: &str,
|
|
message: &str,
|
|
) -> Result<(), axum::Error> {
|
|
let err = DaemonInbound::Error(ErrorInfo {
|
|
code: code.into(),
|
|
message: message.into(),
|
|
});
|
|
let text = serde_json::to_string(&err).expect("serialize error");
|
|
sink.send(Message::Text(text)).await
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
// `dispatch_text` returning `false` is what breaks the handler loop and
|
|
// triggers a deregister (catalog wipe). A frame the registry can't act on
|
|
// must NOT do that, or a newer daemon emitting an unknown variant gets
|
|
// stuck in a tight reconnect loop (NAP-269).
|
|
|
|
#[tokio::test]
|
|
async fn unknown_frame_keeps_connection_alive() {
|
|
let registry = Registry::new();
|
|
let frame = serde_json::json!({ "type": "FutureVariant", "x": 1 }).to_string();
|
|
assert!(
|
|
dispatch_text(®istry, "laptop", "tok", &frame).await,
|
|
"unknown frame type must be skipped, not drop the WS"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn malformed_frame_keeps_connection_alive() {
|
|
let registry = Registry::new();
|
|
assert!(
|
|
dispatch_text(®istry, "laptop", "tok", "{ not valid json").await,
|
|
"malformed frame must be skipped, not drop the WS"
|
|
);
|
|
}
|
|
}
|