feat(maestro): registry fan-out of catalog events to daemons (NAP-246)
Each daemon connection now also subscribes to the registry's event broadcast. A per-connection task pumps peer-filtered events through the existing outbound mpsc as `CatalogEvent` frames; the daemon side mirrors them locally (NAP-246). Two pre-existing race fixes that NAP-246's extra traffic exposed: * `create_session` / `kill_session` now eagerly mirror Ok results into state.sessions (via the same `add_session` / `remove_session` calls the daemon's async `SessionAdded` / `SessionRemoved` will hit later — both are idempotent). HTTP callers no longer see a transient empty catalog immediately after a successful POST. * Catalog frames flow through the same select! arm as RPC frames so the WS sink+stream coupling axum imposes doesn't stall the inbound reader. Updates daemon_ws.rs tests to skip CatalogEvent frames when asserting on RPC frames. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,8 @@ use std::time::Duration;
|
||||
|
||||
use axum::extract::ws::{Message, WebSocket};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use maestro_protocol::{DaemonInbound, DaemonOutbound, ErrorInfo};
|
||||
use maestro_protocol::{DaemonInbound, DaemonOutbound, ErrorInfo, RegistryEvent};
|
||||
use tokio::sync::broadcast::error::RecvError;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::state::Registry;
|
||||
@@ -58,7 +59,13 @@ pub async fn handle_socket(
|
||||
|
||||
let (out_tx, mut out_rx) = mpsc::channel::<DaemonInbound>(OUTBOUND_BUFFER);
|
||||
registry
|
||||
.register(daemon_id.clone(), hostname, version, daemon_url, out_tx)
|
||||
.register(
|
||||
daemon_id.clone(),
|
||||
hostname,
|
||||
version,
|
||||
daemon_url,
|
||||
out_tx.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let _ = sink
|
||||
@@ -67,6 +74,44 @@ pub async fn handle_socket(
|
||||
))
|
||||
.await;
|
||||
|
||||
// NAP-246: forward peer-filtered catalog deltas to the daemon. Initial
|
||||
// snapshot is pushed eagerly; live deltas flow through `catalog_task` →
|
||||
// `out_tx` so they share the existing outbound writer in the select! loop
|
||||
// (any extra sink writer would contend with the WS sink+stream coupling
|
||||
// axum imposes and stall the inbound reader — see history of this file).
|
||||
let (snapshot, mut catalog_rx) = registry.subscribe().await;
|
||||
if let Some(peer_snapshot) = snapshot.for_peer(&daemon_id) {
|
||||
if out_tx
|
||||
.send(DaemonInbound::CatalogEvent {
|
||||
event: peer_snapshot,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
registry.deregister(&daemon_id).await;
|
||||
return Some(daemon_id);
|
||||
}
|
||||
}
|
||||
let catalog_id = daemon_id.clone();
|
||||
let catalog_out = out_tx.clone();
|
||||
let catalog_task = tokio::spawn(async move {
|
||||
loop {
|
||||
let ev = match catalog_rx.recv().await {
|
||||
Ok(e) => e.for_peer(&catalog_id),
|
||||
Err(RecvError::Lagged(n)) => Some(RegistryEvent::Lagged { skipped: n }),
|
||||
Err(RecvError::Closed) => return,
|
||||
};
|
||||
let Some(event) = ev else { continue };
|
||||
if catalog_out
|
||||
.send(DaemonInbound::CatalogEvent { event })
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
inbound = tokio::time::timeout(heartbeat_timeout, stream.next()) => {
|
||||
@@ -83,13 +128,10 @@ pub async fn handle_socket(
|
||||
Ok(Some(Ok(Message::Close(_)))) => break,
|
||||
Ok(Some(Ok(Message::Text(text)))) => {
|
||||
if !dispatch_text(®istry, &daemon_id, &text).await {
|
||||
// malformed; close the connection
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(Some(Ok(Message::Binary(_) | Message::Ping(_) | Message::Pong(_)))) => {
|
||||
// Daemons only send text frames on this endpoint.
|
||||
}
|
||||
Ok(Some(Ok(Message::Binary(_) | Message::Ping(_) | Message::Pong(_)))) => {}
|
||||
}
|
||||
}
|
||||
Some(msg) = out_rx.recv() => {
|
||||
@@ -101,6 +143,7 @@ pub async fn handle_socket(
|
||||
}
|
||||
}
|
||||
|
||||
catalog_task.abort();
|
||||
registry.deregister(&daemon_id).await;
|
||||
Some(daemon_id)
|
||||
}
|
||||
|
||||
@@ -329,9 +329,19 @@ impl Registry {
|
||||
self.inner.lock().await.pending_create.remove(&reply_to);
|
||||
return Err(RegistryError::DaemonGone);
|
||||
}
|
||||
await_rpc(rx, reply_to, |s| &mut s.pending_create, &self.inner)
|
||||
let rpc = await_rpc(rx, reply_to, |s| &mut s.pending_create, &self.inner)
|
||||
.await
|
||||
.map(rpc_into_result)
|
||||
.map(rpc_into_result)?;
|
||||
// Eagerly mirror the session into the catalog on success — the daemon
|
||||
// also emits `SessionAdded` asynchronously, but the HTTP caller would
|
||||
// otherwise see an empty `GET /api/sessions` immediately after a
|
||||
// successful POST (NAP-246 made the race wider by adding extra
|
||||
// outbound traffic on the daemon control link). `add_session` is
|
||||
// idempotent so a later `SessionAdded` for the same id is a no-op.
|
||||
if let Ok(ref session) = rpc {
|
||||
self.add_session(daemon_id, session.clone()).await;
|
||||
}
|
||||
Ok(rpc)
|
||||
}
|
||||
|
||||
/// Forward a `KillSession` to the named daemon and wait for the matching
|
||||
@@ -360,9 +370,16 @@ impl Registry {
|
||||
self.inner.lock().await.pending_kill.remove(&reply_to);
|
||||
return Err(RegistryError::DaemonGone);
|
||||
}
|
||||
await_rpc(rx, reply_to, |s| &mut s.pending_kill, &self.inner)
|
||||
let rpc = await_rpc(rx, reply_to, |s| &mut s.pending_kill, &self.inner)
|
||||
.await
|
||||
.map(rpc_into_result)
|
||||
.map(rpc_into_result)?;
|
||||
// Mirror of the create-path eager update: drop the session locally so
|
||||
// subsequent reads see consistent state. The daemon also emits
|
||||
// `SessionRemoved`; `remove_session` is idempotent.
|
||||
if rpc.is_ok() {
|
||||
self.remove_session(daemon_id, id).await;
|
||||
}
|
||||
Ok(rpc)
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -70,7 +70,13 @@ async fn send_json(ws: &mut WsClient, msg: &DaemonOutbound) {
|
||||
async fn recv_inbound(ws: &mut WsClient) -> Option<DaemonInbound> {
|
||||
while let Some(Ok(msg)) = ws.next().await {
|
||||
if let WsMessage::Text(t) = msg {
|
||||
return Some(serde_json::from_str(&t).unwrap());
|
||||
let parsed: DaemonInbound = serde_json::from_str(&t).unwrap();
|
||||
// NAP-246: skip catalog frames — these are informational and not
|
||||
// what most tests are asserting on.
|
||||
if matches!(parsed, DaemonInbound::CatalogEvent { .. }) {
|
||||
continue;
|
||||
}
|
||||
return Some(parsed);
|
||||
}
|
||||
}
|
||||
None
|
||||
|
||||
Reference in New Issue
Block a user