fix(maestro): remove pending-RPC entries on caller cancellation (NAP-259)
ci / rust (push) Successful in 7m22s

`BrokerHandle::resolve_attach` (daemon) and `Registry::create_session` /
`kill_session` (registry) insert a pending oneshot into a map, then await
the reply. If the awaiting future is cancelled — e.g. an axum handler
dropped because the browser closed the tab or aborted the XHR — the entry
was never removed, leaking one map slot per cancelled request and growing
unboundedly under UI-driven retry traffic.

Introduce a small RAII `PendingGuard` that removes the entry on drop, so
cleanup happens on every exit path including cancellation. Because a sync
`Drop` can't lock a tokio mutex, move the pending maps to
`std::sync::Mutex` (only ever locked for a tiny insert/remove, never held
across an await): broker `pending`, and the registry's `pending_create` /
`pending_kill` lifted out of the async-locked `State`. Removal is
idempotent, so the happy path (reply already taken by `complete_*`) and
the guard's drop don't conflict; `await_rpc` no longer needs its manual
timeout cleanup.

Tested: cancelling an in-flight `resolve_attach` (broker) and
`kill_session` (registry) leaves the pending map empty.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-31 21:10:10 +00:00
co-authored by Claude Opus 4.8
parent 599f3ed178
commit 287afa26ea
2 changed files with 179 additions and 48 deletions
+68 -10
View File
@@ -6,13 +6,33 @@
//! send and the reply correlation live here so the handler stays thin.
use std::collections::HashMap;
use std::sync::Arc;
use std::hash::Hash;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use maestro_protocol::{AttachCandidates, DaemonOutbound, ReplyTo, RpcResult};
use tokio::sync::{mpsc, oneshot, Mutex};
use tokio::sync::{mpsc, oneshot};
use uuid::Uuid;
/// RAII cleanup for a pending-RPC entry. Removing the entry on drop guarantees
/// it is gone on *every* exit path of the awaiting call — success, error,
/// timeout, and crucially future cancellation (the axum handler is dropped when
/// the client hangs up) — so the map can't grow without bound (NAP-259).
///
/// The map is a `std::sync::Mutex` (never held across an await) precisely so the
/// sync `Drop` can lock it. Removal is idempotent: on the happy path the reply
/// handler has already taken the entry and this is a harmless no-op.
struct PendingGuard<'a, K: Eq + Hash, V> {
map: &'a Mutex<HashMap<K, V>>,
key: K,
}
impl<K: Eq + Hash, V> Drop for PendingGuard<'_, K, V> {
fn drop(&mut self) {
self.map.lock().unwrap().remove(&self.key);
}
}
/// How long to wait for a `ResolveAttachResult` after the request goes out.
/// Matches the registry's RPC timeout — a healthy registry replies in ms.
pub const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10);
@@ -55,7 +75,13 @@ impl BrokerHandle {
pub async fn resolve_attach(&self, session_id: Uuid) -> Result<AttachCandidates, BrokerError> {
let reply_to = Uuid::new_v4();
let (tx, rx) = oneshot::channel();
self.pending.lock().await.insert(reply_to, tx);
self.pending.lock().unwrap().insert(reply_to, tx);
// From here on, every exit path (including cancellation of this future)
// removes the pending entry when `_guard` drops (NAP-259).
let _guard = PendingGuard {
map: &*self.pending,
key: reply_to,
};
if self
.out_tx
@@ -66,7 +92,6 @@ impl BrokerHandle {
.await
.is_err()
{
self.pending.lock().await.remove(&reply_to);
return Err(BrokerError::NoControlLink);
}
@@ -76,10 +101,7 @@ impl BrokerHandle {
Err(BrokerError::Registry(error.code, error.message))
}
Ok(Err(_)) => Err(BrokerError::ReplyClosed),
Err(_) => {
self.pending.lock().await.remove(&reply_to);
Err(BrokerError::Timeout(RESOLVE_TIMEOUT))
}
Err(_) => Err(BrokerError::Timeout(RESOLVE_TIMEOUT)),
}
}
@@ -91,8 +113,8 @@ impl BrokerHandle {
reply_to: ReplyTo,
result: RpcResult<AttachCandidates>,
) -> bool {
let mut pending = self.pending.lock().await;
match pending.remove(&reply_to) {
let entry = self.pending.lock().unwrap().remove(&reply_to);
match entry {
Some(tx) => tx.send(result).is_ok(),
None => {
tracing::debug!(?reply_to, "no waiter for ResolveAttachResult");
@@ -101,3 +123,39 @@ impl BrokerHandle {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Cancelling (dropping) an in-flight `resolve_attach` must not leak its
/// pending-reply entry — otherwise a browser hanging up mid-attach grows
/// the map without bound (NAP-259).
#[tokio::test]
async fn cancelled_resolve_attach_removes_pending_entry() {
// Keep the receiver alive so `out_tx.send()` succeeds and the call
// parks on the reply oneshot rather than failing fast.
let (handle, _out_rx) = BrokerHandle::new();
{
let fut = handle.resolve_attach(Uuid::new_v4());
tokio::pin!(fut);
tokio::select! {
biased;
_ = &mut fut => panic!("resolve must not complete without a reply"),
_ = tokio::time::sleep(Duration::from_millis(20)) => {}
}
assert_eq!(
handle.pending.lock().unwrap().len(),
1,
"entry should be in-flight before cancellation"
);
}
assert_eq!(
handle.pending.lock().unwrap().len(),
0,
"pending entry must be removed on cancellation (NAP-259)"
);
}
}