fix(maestro): remove pending-RPC entries on caller cancellation (NAP-259)
ci / rust (push) Successful in 7m22s
ci / rust (push) Successful in 7m22s
`BrokerHandle::resolve_attach` (daemon) and `Registry::create_session` / `kill_session` (registry) insert a pending oneshot into a map, then await the reply. If the awaiting future is cancelled — e.g. an axum handler dropped because the browser closed the tab or aborted the XHR — the entry was never removed, leaking one map slot per cancelled request and growing unboundedly under UI-driven retry traffic. Introduce a small RAII `PendingGuard` that removes the entry on drop, so cleanup happens on every exit path including cancellation. Because a sync `Drop` can't lock a tokio mutex, move the pending maps to `std::sync::Mutex` (only ever locked for a tiny insert/remove, never held across an await): broker `pending`, and the registry's `pending_create` / `pending_kill` lifted out of the async-locked `State`. Removal is idempotent, so the happy path (reply already taken by `complete_*`) and the guard's drop don't conflict; `await_rpc` no longer needs its manual timeout cleanup. Tested: cancelling an in-flight `resolve_attach` (broker) and `kill_session` (registry) leaves the pending map empty. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6,13 +6,33 @@
|
||||
//! send and the reply correlation live here so the handler stays thin.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::hash::Hash;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use maestro_protocol::{AttachCandidates, DaemonOutbound, ReplyTo, RpcResult};
|
||||
use tokio::sync::{mpsc, oneshot, Mutex};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// RAII cleanup for a pending-RPC entry. Removing the entry on drop guarantees
|
||||
/// it is gone on *every* exit path of the awaiting call — success, error,
|
||||
/// timeout, and crucially future cancellation (the axum handler is dropped when
|
||||
/// the client hangs up) — so the map can't grow without bound (NAP-259).
|
||||
///
|
||||
/// The map is a `std::sync::Mutex` (never held across an await) precisely so the
|
||||
/// sync `Drop` can lock it. Removal is idempotent: on the happy path the reply
|
||||
/// handler has already taken the entry and this is a harmless no-op.
|
||||
struct PendingGuard<'a, K: Eq + Hash, V> {
|
||||
map: &'a Mutex<HashMap<K, V>>,
|
||||
key: K,
|
||||
}
|
||||
|
||||
impl<K: Eq + Hash, V> Drop for PendingGuard<'_, K, V> {
|
||||
fn drop(&mut self) {
|
||||
self.map.lock().unwrap().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
/// How long to wait for a `ResolveAttachResult` after the request goes out.
|
||||
/// Matches the registry's RPC timeout — a healthy registry replies in ms.
|
||||
pub const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
@@ -55,7 +75,13 @@ impl BrokerHandle {
|
||||
pub async fn resolve_attach(&self, session_id: Uuid) -> Result<AttachCandidates, BrokerError> {
|
||||
let reply_to = Uuid::new_v4();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.pending.lock().await.insert(reply_to, tx);
|
||||
self.pending.lock().unwrap().insert(reply_to, tx);
|
||||
// From here on, every exit path (including cancellation of this future)
|
||||
// removes the pending entry when `_guard` drops (NAP-259).
|
||||
let _guard = PendingGuard {
|
||||
map: &*self.pending,
|
||||
key: reply_to,
|
||||
};
|
||||
|
||||
if self
|
||||
.out_tx
|
||||
@@ -66,7 +92,6 @@ impl BrokerHandle {
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
self.pending.lock().await.remove(&reply_to);
|
||||
return Err(BrokerError::NoControlLink);
|
||||
}
|
||||
|
||||
@@ -76,10 +101,7 @@ impl BrokerHandle {
|
||||
Err(BrokerError::Registry(error.code, error.message))
|
||||
}
|
||||
Ok(Err(_)) => Err(BrokerError::ReplyClosed),
|
||||
Err(_) => {
|
||||
self.pending.lock().await.remove(&reply_to);
|
||||
Err(BrokerError::Timeout(RESOLVE_TIMEOUT))
|
||||
}
|
||||
Err(_) => Err(BrokerError::Timeout(RESOLVE_TIMEOUT)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,8 +113,8 @@ impl BrokerHandle {
|
||||
reply_to: ReplyTo,
|
||||
result: RpcResult<AttachCandidates>,
|
||||
) -> bool {
|
||||
let mut pending = self.pending.lock().await;
|
||||
match pending.remove(&reply_to) {
|
||||
let entry = self.pending.lock().unwrap().remove(&reply_to);
|
||||
match entry {
|
||||
Some(tx) => tx.send(result).is_ok(),
|
||||
None => {
|
||||
tracing::debug!(?reply_to, "no waiter for ResolveAttachResult");
|
||||
@@ -101,3 +123,39 @@ impl BrokerHandle {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Cancelling (dropping) an in-flight `resolve_attach` must not leak its
|
||||
/// pending-reply entry — otherwise a browser hanging up mid-attach grows
|
||||
/// the map without bound (NAP-259).
|
||||
#[tokio::test]
|
||||
async fn cancelled_resolve_attach_removes_pending_entry() {
|
||||
// Keep the receiver alive so `out_tx.send()` succeeds and the call
|
||||
// parks on the reply oneshot rather than failing fast.
|
||||
let (handle, _out_rx) = BrokerHandle::new();
|
||||
|
||||
{
|
||||
let fut = handle.resolve_attach(Uuid::new_v4());
|
||||
tokio::pin!(fut);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut fut => panic!("resolve must not complete without a reply"),
|
||||
_ = tokio::time::sleep(Duration::from_millis(20)) => {}
|
||||
}
|
||||
assert_eq!(
|
||||
handle.pending.lock().unwrap().len(),
|
||||
1,
|
||||
"entry should be in-flight before cancellation"
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
handle.pending.lock().unwrap().len(),
|
||||
0,
|
||||
"pending entry must be removed on cancellation (NAP-259)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user