feat(maestro): reap dead tmux sessions via liveness watcher (NAP-227)
ci / rust (push) Successful in 1m37s

The daemon never noticed when a session's command exited and tmux tore
the session down — the in-memory `SessionManager` map plus the registry
catalog kept showing it as `Running`, and a follow-up `maestro attach`
would hang on an empty broadcast.

Adds a per-host background task that polls `tmux list-sessions` every
2s, diffs against the in-memory set, and reaps anything we still track
but tmux has lost: drops the record, deletes the sidecar + FIFO,
cleans up its worktree if requested, and emits `SessionEvent::Removed`
so the registry purges its catalog.

The task holds a `Weak<Inner>` so it exits cleanly once the last
`SessionManager` clone is dropped. Skips the tmux subprocess when the
in-memory map is empty.
This commit is contained in:
2026-05-26 07:26:42 +00:00
parent f0b01ae295
commit a2ffb0ad33
3 changed files with 169 additions and 2 deletions
+2
View File
@@ -9,6 +9,7 @@ use std::path::PathBuf;
use std::process::ExitCode;
use clap::{Parser, Subcommand};
use maestro_daemon::session::DEFAULT_LIVENESS_INTERVAL;
use maestro_daemon::sidecar::SidecarStore;
use maestro_daemon::tmux::Tmux;
use maestro_daemon::worktree::WorktreeManager;
@@ -128,6 +129,7 @@ async fn run(config_override: Option<PathBuf>) -> Result<(), Box<dyn Error>> {
let recovered = manager.recover().await?;
tracing::info!(recovered, "adopted existing tmux sessions");
manager.start_liveness_watcher(DEFAULT_LIVENESS_INTERVAL);
let verifier = JwtVerifier::new(&config.token, &config.daemon_id);
let router = attach::router(manager.clone(), verifier);
+85 -1
View File
@@ -5,9 +5,10 @@
//! single point through which the registry client and the per-session WS
//! handler interact with tmux.
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use chrono::Utc;
use maestro_protocol::{CreateSessionRequest, ErrorInfo, Session, SessionStatus, WorktreeSpec};
@@ -24,6 +25,9 @@ use crate::worktree::{WorktreeError, WorktreeManager};
/// reconciles on `Hello` via the snapshot).
pub const EVENTS_CAPACITY: usize = 256;
/// How often the liveness watcher polls tmux for sessions that disappeared.
pub const DEFAULT_LIVENESS_INTERVAL: Duration = Duration::from_secs(2);
#[derive(Debug, Clone)]
pub enum SessionEvent {
Added(Session),
@@ -242,6 +246,31 @@ impl SessionManager {
Ok(())
}
/// Spawn a background task that polls tmux every `interval` and reaps any
/// in-memory sessions whose underlying tmux session has disappeared (the
/// command exited, the user ran `tmux kill-session`, etc.).
///
/// The task holds a `Weak<Inner>` and exits cleanly once the last
/// `SessionManager` clone is dropped — call this once from the daemon
/// startup path after [`Self::recover`] has populated the map.
pub fn start_liveness_watcher(&self, interval: Duration) {
let weak = Arc::downgrade(&self.inner);
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
// first tick fires immediately — skip it so we don't reap before
// the caller has a chance to create anything.
ticker.tick().await;
loop {
ticker.tick().await;
let Some(inner) = weak.upgrade() else {
return;
};
run_liveness_pass(&inner).await;
}
});
}
/// Re-attach to tmux sessions left behind by a previous daemon process.
///
/// For each `maestro-<uuid>` session we read its sidecar (or synthesise
@@ -312,3 +341,58 @@ impl SessionManager {
.map_err(SessionError::Tmux)
}
}
/// One pass of the liveness watcher: diff the in-memory session set against
/// the tmux-reported one, and reap anything we still track but tmux has lost.
async fn run_liveness_pass(inner: &Arc<Inner>) {
// Snapshot the in-memory ids first; if we own no sessions there is
// nothing to reap and no reason to spawn a tmux subprocess (this also
// avoids racing with the very first `new-session` that would otherwise
// be competing to start the tmux server).
let known: Vec<Uuid> = {
let sessions = inner.sessions.lock().await;
if sessions.is_empty() {
return;
}
sessions.keys().copied().collect()
};
let live_ids = match inner.tmux.list_sessions() {
Ok(ids) => ids,
Err(e) => {
tracing::warn!(error = ?e, "liveness watcher: tmux list-sessions failed");
return;
}
};
let live: HashSet<Uuid> = live_ids.into_iter().collect();
for id in known {
if !live.contains(&id) {
reap_dead_session(inner, id).await;
}
}
}
/// Remove a session whose tmux session is already gone: drop the in-memory
/// record, delete the sidecar + FIFO, clean up its worktree if requested, and
/// fire `SessionEvent::Removed`. No tmux kill — there is nothing left to kill.
async fn reap_dead_session(inner: &Arc<Inner>, id: Uuid) {
let record = {
let mut sessions = inner.sessions.lock().await;
sessions.remove(&id)
};
let Some(record) = record else {
return;
};
tracing::info!(%id, "session vanished from tmux — reaping");
let _ = std::fs::remove_file(&record.fifo_path);
if let Err(e) = inner.sidecar.delete(id) {
tracing::warn!(error = ?e, %id, "reap: failed to delete sidecar");
}
if record.cleanup_worktree_on_kill {
if let (Some(base), Some(path)) = (&record.worktree_base_repo, &record.worktree_path) {
let _ = inner.worktree_mgr.remove(base, path);
}
}
let _ = inner.events.send(SessionEvent::Removed(id));
}
+82 -1
View File
@@ -6,7 +6,8 @@ use std::collections::HashMap;
use std::path::PathBuf;
use std::time::Duration;
use common::DaemonFixture;
use common::{wait_for_async, DaemonFixture};
use maestro_daemon::SessionEvent;
use maestro_protocol::CreateSessionRequest;
use tempfile::TempDir;
@@ -157,3 +158,83 @@ async fn pty_broadcast_delivers_bytes_to_subscribers() {
fx.manager.kill(s.id).await.unwrap();
let _ = (cwd, PathBuf::from("/"));
}
#[tokio::test]
async fn liveness_watcher_reaps_dead_sessions_and_emits_removed() {
let fx = DaemonFixture::new("laptop");
let cwd = TempDir::new().unwrap();
let s = fx
.manager
.create(CreateSessionRequest {
daemon_id: "laptop".into(),
cwd: cwd.path().to_path_buf(),
command: "sleep 30".into(),
env: HashMap::new(),
metadata: HashMap::new(),
worktree: None,
})
.await
.expect("create");
// Subscribe *after* the create so we don't have to filter the Added event.
let mut events = fx.manager.events();
fx.manager.start_liveness_watcher(Duration::from_millis(50));
// Kill the tmux session out from under the daemon. This is the scenario
// the watcher exists for: command exits, tmux destroys the session, the
// daemon never finds out through its normal channels.
fx.tmux.tmux.kill_session(s.id).expect("kill tmux directly");
// Within a few watcher ticks the daemon should drop the session.
wait_for_async(Duration::from_secs(3), || async {
!fx.manager.has(s.id).await
})
.await;
// Sidecar + FIFO should be cleaned up too.
assert!(!fx.sidecar_dir().join(format!("{}.json", s.id)).exists());
assert!(!fx.fifo_dir().join(format!("{}.fifo", s.id)).exists());
// And we should have seen a Removed event for this id.
let mut saw_removed = false;
let deadline = std::time::Instant::now() + Duration::from_secs(1);
while std::time::Instant::now() < deadline && !saw_removed {
match tokio::time::timeout(Duration::from_millis(50), events.recv()).await {
Ok(Ok(SessionEvent::Removed(id))) if id == s.id => saw_removed = true,
Ok(Ok(_)) => continue,
_ => break,
}
}
assert!(
saw_removed,
"expected SessionEvent::Removed for dead session"
);
}
#[tokio::test]
async fn liveness_watcher_leaves_running_sessions_alone() {
let fx = DaemonFixture::new("laptop");
let cwd = TempDir::new().unwrap();
let s = fx
.manager
.create(CreateSessionRequest {
daemon_id: "laptop".into(),
cwd: cwd.path().to_path_buf(),
command: "sleep 30".into(),
env: HashMap::new(),
metadata: HashMap::new(),
worktree: None,
})
.await
.expect("create");
fx.manager.start_liveness_watcher(Duration::from_millis(50));
// Give the watcher enough time to run a handful of ticks.
tokio::time::sleep(Duration::from_millis(400)).await;
assert!(fx.manager.has(s.id).await, "live session got reaped");
fx.manager.kill(s.id).await.unwrap();
}