feat(maestro): reap dead tmux sessions via liveness watcher (NAP-227)
ci / rust (push) Successful in 1m37s
ci / rust (push) Successful in 1m37s
The daemon never noticed when a session's command exited and tmux tore the session down — the in-memory `SessionManager` map plus the registry catalog kept showing it as `Running`, and a follow-up `maestro attach` would hang on an empty broadcast. Adds a per-host background task that polls `tmux list-sessions` every 2s, diffs against the in-memory set, and reaps anything we still track but tmux has lost: drops the record, deletes the sidecar + FIFO, cleans up its worktree if requested, and emits `SessionEvent::Removed` so the registry purges its catalog. The task holds a `Weak<Inner>` so it exits cleanly once the last `SessionManager` clone is dropped. Skips the tmux subprocess when the in-memory map is empty.
This commit is contained in:
@@ -9,6 +9,7 @@ use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use maestro_daemon::session::DEFAULT_LIVENESS_INTERVAL;
|
||||
use maestro_daemon::sidecar::SidecarStore;
|
||||
use maestro_daemon::tmux::Tmux;
|
||||
use maestro_daemon::worktree::WorktreeManager;
|
||||
@@ -128,6 +129,7 @@ async fn run(config_override: Option<PathBuf>) -> Result<(), Box<dyn Error>> {
|
||||
|
||||
let recovered = manager.recover().await?;
|
||||
tracing::info!(recovered, "adopted existing tmux sessions");
|
||||
manager.start_liveness_watcher(DEFAULT_LIVENESS_INTERVAL);
|
||||
|
||||
let verifier = JwtVerifier::new(&config.token, &config.daemon_id);
|
||||
let router = attach::router(manager.clone(), verifier);
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
//! single point through which the registry client and the per-session WS
|
||||
//! handler interact with tmux.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
use maestro_protocol::{CreateSessionRequest, ErrorInfo, Session, SessionStatus, WorktreeSpec};
|
||||
@@ -24,6 +25,9 @@ use crate::worktree::{WorktreeError, WorktreeManager};
|
||||
/// reconciles on `Hello` via the snapshot).
|
||||
pub const EVENTS_CAPACITY: usize = 256;
|
||||
|
||||
/// How often the liveness watcher polls tmux for sessions that disappeared.
|
||||
pub const DEFAULT_LIVENESS_INTERVAL: Duration = Duration::from_secs(2);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum SessionEvent {
|
||||
Added(Session),
|
||||
@@ -242,6 +246,31 @@ impl SessionManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Spawn a background task that polls tmux every `interval` and reaps any
|
||||
/// in-memory sessions whose underlying tmux session has disappeared (the
|
||||
/// command exited, the user ran `tmux kill-session`, etc.).
|
||||
///
|
||||
/// The task holds a `Weak<Inner>` and exits cleanly once the last
|
||||
/// `SessionManager` clone is dropped — call this once from the daemon
|
||||
/// startup path after [`Self::recover`] has populated the map.
|
||||
pub fn start_liveness_watcher(&self, interval: Duration) {
|
||||
let weak = Arc::downgrade(&self.inner);
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(interval);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
// first tick fires immediately — skip it so we don't reap before
|
||||
// the caller has a chance to create anything.
|
||||
ticker.tick().await;
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
let Some(inner) = weak.upgrade() else {
|
||||
return;
|
||||
};
|
||||
run_liveness_pass(&inner).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Re-attach to tmux sessions left behind by a previous daemon process.
|
||||
///
|
||||
/// For each `maestro-<uuid>` session we read its sidecar (or synthesise
|
||||
@@ -312,3 +341,58 @@ impl SessionManager {
|
||||
.map_err(SessionError::Tmux)
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass of the liveness watcher: diff the in-memory session set against
|
||||
/// the tmux-reported one, and reap anything we still track but tmux has lost.
|
||||
async fn run_liveness_pass(inner: &Arc<Inner>) {
|
||||
// Snapshot the in-memory ids first; if we own no sessions there is
|
||||
// nothing to reap and no reason to spawn a tmux subprocess (this also
|
||||
// avoids racing with the very first `new-session` that would otherwise
|
||||
// be competing to start the tmux server).
|
||||
let known: Vec<Uuid> = {
|
||||
let sessions = inner.sessions.lock().await;
|
||||
if sessions.is_empty() {
|
||||
return;
|
||||
}
|
||||
sessions.keys().copied().collect()
|
||||
};
|
||||
|
||||
let live_ids = match inner.tmux.list_sessions() {
|
||||
Ok(ids) => ids,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "liveness watcher: tmux list-sessions failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let live: HashSet<Uuid> = live_ids.into_iter().collect();
|
||||
|
||||
for id in known {
|
||||
if !live.contains(&id) {
|
||||
reap_dead_session(inner, id).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a session whose tmux session is already gone: drop the in-memory
|
||||
/// record, delete the sidecar + FIFO, clean up its worktree if requested, and
|
||||
/// fire `SessionEvent::Removed`. No tmux kill — there is nothing left to kill.
|
||||
async fn reap_dead_session(inner: &Arc<Inner>, id: Uuid) {
|
||||
let record = {
|
||||
let mut sessions = inner.sessions.lock().await;
|
||||
sessions.remove(&id)
|
||||
};
|
||||
let Some(record) = record else {
|
||||
return;
|
||||
};
|
||||
tracing::info!(%id, "session vanished from tmux — reaping");
|
||||
let _ = std::fs::remove_file(&record.fifo_path);
|
||||
if let Err(e) = inner.sidecar.delete(id) {
|
||||
tracing::warn!(error = ?e, %id, "reap: failed to delete sidecar");
|
||||
}
|
||||
if record.cleanup_worktree_on_kill {
|
||||
if let (Some(base), Some(path)) = (&record.worktree_base_repo, &record.worktree_path) {
|
||||
let _ = inner.worktree_mgr.remove(base, path);
|
||||
}
|
||||
}
|
||||
let _ = inner.events.send(SessionEvent::Removed(id));
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@ use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::DaemonFixture;
|
||||
use common::{wait_for_async, DaemonFixture};
|
||||
use maestro_daemon::SessionEvent;
|
||||
use maestro_protocol::CreateSessionRequest;
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -157,3 +158,83 @@ async fn pty_broadcast_delivers_bytes_to_subscribers() {
|
||||
fx.manager.kill(s.id).await.unwrap();
|
||||
let _ = (cwd, PathBuf::from("/"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn liveness_watcher_reaps_dead_sessions_and_emits_removed() {
|
||||
let fx = DaemonFixture::new("laptop");
|
||||
let cwd = TempDir::new().unwrap();
|
||||
|
||||
let s = fx
|
||||
.manager
|
||||
.create(CreateSessionRequest {
|
||||
daemon_id: "laptop".into(),
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
command: "sleep 30".into(),
|
||||
env: HashMap::new(),
|
||||
metadata: HashMap::new(),
|
||||
worktree: None,
|
||||
})
|
||||
.await
|
||||
.expect("create");
|
||||
|
||||
// Subscribe *after* the create so we don't have to filter the Added event.
|
||||
let mut events = fx.manager.events();
|
||||
fx.manager.start_liveness_watcher(Duration::from_millis(50));
|
||||
|
||||
// Kill the tmux session out from under the daemon. This is the scenario
|
||||
// the watcher exists for: command exits, tmux destroys the session, the
|
||||
// daemon never finds out through its normal channels.
|
||||
fx.tmux.tmux.kill_session(s.id).expect("kill tmux directly");
|
||||
|
||||
// Within a few watcher ticks the daemon should drop the session.
|
||||
wait_for_async(Duration::from_secs(3), || async {
|
||||
!fx.manager.has(s.id).await
|
||||
})
|
||||
.await;
|
||||
|
||||
// Sidecar + FIFO should be cleaned up too.
|
||||
assert!(!fx.sidecar_dir().join(format!("{}.json", s.id)).exists());
|
||||
assert!(!fx.fifo_dir().join(format!("{}.fifo", s.id)).exists());
|
||||
|
||||
// And we should have seen a Removed event for this id.
|
||||
let mut saw_removed = false;
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(1);
|
||||
while std::time::Instant::now() < deadline && !saw_removed {
|
||||
match tokio::time::timeout(Duration::from_millis(50), events.recv()).await {
|
||||
Ok(Ok(SessionEvent::Removed(id))) if id == s.id => saw_removed = true,
|
||||
Ok(Ok(_)) => continue,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_removed,
|
||||
"expected SessionEvent::Removed for dead session"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn liveness_watcher_leaves_running_sessions_alone() {
|
||||
let fx = DaemonFixture::new("laptop");
|
||||
let cwd = TempDir::new().unwrap();
|
||||
|
||||
let s = fx
|
||||
.manager
|
||||
.create(CreateSessionRequest {
|
||||
daemon_id: "laptop".into(),
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
command: "sleep 30".into(),
|
||||
env: HashMap::new(),
|
||||
metadata: HashMap::new(),
|
||||
worktree: None,
|
||||
})
|
||||
.await
|
||||
.expect("create");
|
||||
|
||||
fx.manager.start_liveness_watcher(Duration::from_millis(50));
|
||||
|
||||
// Give the watcher enough time to run a handful of ticks.
|
||||
tokio::time::sleep(Duration::from_millis(400)).await;
|
||||
assert!(fx.manager.has(s.id).await, "live session got reaped");
|
||||
|
||||
fx.manager.kill(s.id).await.unwrap();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user