From a2ffb0ad331369eea151542612fda899e683acdd Mon Sep 17 00:00:00 2001 From: naps62-yolo Date: Tue, 26 May 2026 07:26:42 +0000 Subject: [PATCH] feat(maestro): reap dead tmux sessions via liveness watcher (NAP-227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daemon never noticed when a session's command exited and tmux tore the session down — the in-memory `SessionManager` map plus the registry catalog kept showing it as `Running`, and a follow-up `maestro attach` would hang on an empty broadcast. Adds a per-host background task that polls `tmux list-sessions` every 2s, diffs against the in-memory set, and reaps anything we still track but tmux has lost: drops the record, deletes the sidecar + FIFO, cleans up its worktree if requested, and emits `SessionEvent::Removed` so the registry purges its catalog. The task holds a `Weak` so it exits cleanly once the last `SessionManager` clone is dropped. Skips the tmux subprocess when the in-memory map is empty. --- crates/maestro-daemon/src/main.rs | 2 + crates/maestro-daemon/src/session.rs | 86 ++++++++++++++++++- .../maestro-daemon/tests/session_manager.rs | 83 +++++++++++++++++- 3 files changed, 169 insertions(+), 2 deletions(-) diff --git a/crates/maestro-daemon/src/main.rs b/crates/maestro-daemon/src/main.rs index 63933e4..acf5fd1 100644 --- a/crates/maestro-daemon/src/main.rs +++ b/crates/maestro-daemon/src/main.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::process::ExitCode; use clap::{Parser, Subcommand}; +use maestro_daemon::session::DEFAULT_LIVENESS_INTERVAL; use maestro_daemon::sidecar::SidecarStore; use maestro_daemon::tmux::Tmux; use maestro_daemon::worktree::WorktreeManager; @@ -128,6 +129,7 @@ async fn run(config_override: Option) -> Result<(), Box> { let recovered = manager.recover().await?; tracing::info!(recovered, "adopted existing tmux sessions"); + manager.start_liveness_watcher(DEFAULT_LIVENESS_INTERVAL); let verifier = JwtVerifier::new(&config.token, &config.daemon_id); let router = attach::router(manager.clone(), verifier); diff --git a/crates/maestro-daemon/src/session.rs b/crates/maestro-daemon/src/session.rs index a304818..3b52111 100644 --- a/crates/maestro-daemon/src/session.rs +++ b/crates/maestro-daemon/src/session.rs @@ -5,9 +5,10 @@ //! single point through which the registry client and the per-session WS //! handler interact with tmux. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use chrono::Utc; use maestro_protocol::{CreateSessionRequest, ErrorInfo, Session, SessionStatus, WorktreeSpec}; @@ -24,6 +25,9 @@ use crate::worktree::{WorktreeError, WorktreeManager}; /// reconciles on `Hello` via the snapshot). pub const EVENTS_CAPACITY: usize = 256; +/// How often the liveness watcher polls tmux for sessions that disappeared. +pub const DEFAULT_LIVENESS_INTERVAL: Duration = Duration::from_secs(2); + #[derive(Debug, Clone)] pub enum SessionEvent { Added(Session), @@ -242,6 +246,31 @@ impl SessionManager { Ok(()) } + /// Spawn a background task that polls tmux every `interval` and reaps any + /// in-memory sessions whose underlying tmux session has disappeared (the + /// command exited, the user ran `tmux kill-session`, etc.). + /// + /// The task holds a `Weak` and exits cleanly once the last + /// `SessionManager` clone is dropped — call this once from the daemon + /// startup path after [`Self::recover`] has populated the map. + pub fn start_liveness_watcher(&self, interval: Duration) { + let weak = Arc::downgrade(&self.inner); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // first tick fires immediately — skip it so we don't reap before + // the caller has a chance to create anything. + ticker.tick().await; + loop { + ticker.tick().await; + let Some(inner) = weak.upgrade() else { + return; + }; + run_liveness_pass(&inner).await; + } + }); + } + /// Re-attach to tmux sessions left behind by a previous daemon process. /// /// For each `maestro-` session we read its sidecar (or synthesise @@ -312,3 +341,58 @@ impl SessionManager { .map_err(SessionError::Tmux) } } + +/// One pass of the liveness watcher: diff the in-memory session set against +/// the tmux-reported one, and reap anything we still track but tmux has lost. +async fn run_liveness_pass(inner: &Arc) { + // Snapshot the in-memory ids first; if we own no sessions there is + // nothing to reap and no reason to spawn a tmux subprocess (this also + // avoids racing with the very first `new-session` that would otherwise + // be competing to start the tmux server). + let known: Vec = { + let sessions = inner.sessions.lock().await; + if sessions.is_empty() { + return; + } + sessions.keys().copied().collect() + }; + + let live_ids = match inner.tmux.list_sessions() { + Ok(ids) => ids, + Err(e) => { + tracing::warn!(error = ?e, "liveness watcher: tmux list-sessions failed"); + return; + } + }; + let live: HashSet = live_ids.into_iter().collect(); + + for id in known { + if !live.contains(&id) { + reap_dead_session(inner, id).await; + } + } +} + +/// Remove a session whose tmux session is already gone: drop the in-memory +/// record, delete the sidecar + FIFO, clean up its worktree if requested, and +/// fire `SessionEvent::Removed`. No tmux kill — there is nothing left to kill. +async fn reap_dead_session(inner: &Arc, id: Uuid) { + let record = { + let mut sessions = inner.sessions.lock().await; + sessions.remove(&id) + }; + let Some(record) = record else { + return; + }; + tracing::info!(%id, "session vanished from tmux — reaping"); + let _ = std::fs::remove_file(&record.fifo_path); + if let Err(e) = inner.sidecar.delete(id) { + tracing::warn!(error = ?e, %id, "reap: failed to delete sidecar"); + } + if record.cleanup_worktree_on_kill { + if let (Some(base), Some(path)) = (&record.worktree_base_repo, &record.worktree_path) { + let _ = inner.worktree_mgr.remove(base, path); + } + } + let _ = inner.events.send(SessionEvent::Removed(id)); +} diff --git a/crates/maestro-daemon/tests/session_manager.rs b/crates/maestro-daemon/tests/session_manager.rs index 2454560..33b1270 100644 --- a/crates/maestro-daemon/tests/session_manager.rs +++ b/crates/maestro-daemon/tests/session_manager.rs @@ -6,7 +6,8 @@ use std::collections::HashMap; use std::path::PathBuf; use std::time::Duration; -use common::DaemonFixture; +use common::{wait_for_async, DaemonFixture}; +use maestro_daemon::SessionEvent; use maestro_protocol::CreateSessionRequest; use tempfile::TempDir; @@ -157,3 +158,83 @@ async fn pty_broadcast_delivers_bytes_to_subscribers() { fx.manager.kill(s.id).await.unwrap(); let _ = (cwd, PathBuf::from("/")); } + +#[tokio::test] +async fn liveness_watcher_reaps_dead_sessions_and_emits_removed() { + let fx = DaemonFixture::new("laptop"); + let cwd = TempDir::new().unwrap(); + + let s = fx + .manager + .create(CreateSessionRequest { + daemon_id: "laptop".into(), + cwd: cwd.path().to_path_buf(), + command: "sleep 30".into(), + env: HashMap::new(), + metadata: HashMap::new(), + worktree: None, + }) + .await + .expect("create"); + + // Subscribe *after* the create so we don't have to filter the Added event. + let mut events = fx.manager.events(); + fx.manager.start_liveness_watcher(Duration::from_millis(50)); + + // Kill the tmux session out from under the daemon. This is the scenario + // the watcher exists for: command exits, tmux destroys the session, the + // daemon never finds out through its normal channels. + fx.tmux.tmux.kill_session(s.id).expect("kill tmux directly"); + + // Within a few watcher ticks the daemon should drop the session. + wait_for_async(Duration::from_secs(3), || async { + !fx.manager.has(s.id).await + }) + .await; + + // Sidecar + FIFO should be cleaned up too. + assert!(!fx.sidecar_dir().join(format!("{}.json", s.id)).exists()); + assert!(!fx.fifo_dir().join(format!("{}.fifo", s.id)).exists()); + + // And we should have seen a Removed event for this id. + let mut saw_removed = false; + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while std::time::Instant::now() < deadline && !saw_removed { + match tokio::time::timeout(Duration::from_millis(50), events.recv()).await { + Ok(Ok(SessionEvent::Removed(id))) if id == s.id => saw_removed = true, + Ok(Ok(_)) => continue, + _ => break, + } + } + assert!( + saw_removed, + "expected SessionEvent::Removed for dead session" + ); +} + +#[tokio::test] +async fn liveness_watcher_leaves_running_sessions_alone() { + let fx = DaemonFixture::new("laptop"); + let cwd = TempDir::new().unwrap(); + + let s = fx + .manager + .create(CreateSessionRequest { + daemon_id: "laptop".into(), + cwd: cwd.path().to_path_buf(), + command: "sleep 30".into(), + env: HashMap::new(), + metadata: HashMap::new(), + worktree: None, + }) + .await + .expect("create"); + + fx.manager.start_liveness_watcher(Duration::from_millis(50)); + + // Give the watcher enough time to run a handful of ticks. + tokio::time::sleep(Duration::from_millis(400)).await; + assert!(fx.manager.has(s.id).await, "live session got reaped"); + + fx.manager.kill(s.id).await.unwrap(); +}