This repository has been archived on 2026-10-08. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
maestro/crates/maestro-daemon/src/session.rs
T
naps62-yolo a2ffb0ad33
ci / rust (push) Successful in 1m37s
feat(maestro): reap dead tmux sessions via liveness watcher (NAP-227)
The daemon never noticed when a session's command exited and tmux tore
the session down — the in-memory `SessionManager` map plus the registry
catalog kept showing it as `Running`, and a follow-up `maestro attach`
would hang on an empty broadcast.

Adds a per-host background task that polls `tmux list-sessions` every
2s, diffs against the in-memory set, and reaps anything we still track
but tmux has lost: drops the record, deletes the sidecar + FIFO,
cleans up its worktree if requested, and emits `SessionEvent::Removed`
so the registry purges its catalog.

The task holds a `Weak<Inner>` so it exits cleanly once the last
`SessionManager` clone is dropped. Skips the tmux subprocess when the
in-memory map is empty.
2026-05-26 07:26:42 +00:00

399 lines
14 KiB
Rust

//! Session manager — high-level orchestration of tmux sessions, sidecars,
//! worktrees, and PTY bridges.
//!
//! Owns the daemon's authoritative in-memory map of live sessions and is the
//! single point through which the registry client and the per-session WS
//! handler interact with tmux.
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use chrono::Utc;
use maestro_protocol::{CreateSessionRequest, ErrorInfo, Session, SessionStatus, WorktreeSpec};
use tokio::sync::{broadcast, Mutex};
use uuid::Uuid;
use crate::pty::{pipe_pane_command, PtyBridge};
use crate::sidecar::{SidecarError, SidecarStore};
use crate::tmux::{Tmux, TmuxError};
use crate::worktree::{WorktreeError, WorktreeManager};
/// Capacity of the session-events broadcast channel. The registry client
/// listens; if it lags by this much we just drop frames (the registry already
/// reconciles on `Hello` via the snapshot).
pub const EVENTS_CAPACITY: usize = 256;
/// How often the liveness watcher polls tmux for sessions that disappeared.
pub const DEFAULT_LIVENESS_INTERVAL: Duration = Duration::from_secs(2);
#[derive(Debug, Clone)]
pub enum SessionEvent {
Added(Session),
Removed(Uuid),
Updated(Session),
}
/// Per-session daemon-local bookkeeping that complements the public `Session`.
#[derive(Debug)]
pub struct SessionRecord {
pub session: Session,
pub fifo_path: PathBuf,
/// Worktree we created at spawn time and that should be cleaned up on
/// kill (if `cleanup_worktree_on_kill`).
pub worktree_path: Option<PathBuf>,
/// `base_repo` from the original `WorktreeSpec`. Needed because `git
/// worktree remove` is invoked from the base repo.
pub worktree_base_repo: Option<PathBuf>,
pub cleanup_worktree_on_kill: bool,
pub bridge: PtyBridge,
}
#[derive(Debug, thiserror::Error)]
pub enum SessionError {
#[error("tmux: {0}")]
Tmux(#[from] TmuxError),
#[error("worktree: {0}")]
Worktree(#[from] WorktreeError),
#[error("sidecar: {0}")]
Sidecar(#[from] SidecarError),
#[error("io: {0}")]
Io(#[from] std::io::Error),
#[error("session not found: {0}")]
NotFound(Uuid),
#[error("request targets daemon `{wanted}`, this daemon is `{ours}`")]
WrongDaemon { wanted: String, ours: String },
}
impl SessionError {
pub fn to_error_info(&self) -> ErrorInfo {
let code = match self {
SessionError::Tmux(_) => "tmux_failed",
SessionError::Worktree(_) => "worktree_failed",
SessionError::Sidecar(_) => "sidecar_failed",
SessionError::Io(_) => "io_failed",
SessionError::NotFound(_) => "not_found",
SessionError::WrongDaemon { .. } => "wrong_daemon",
};
ErrorInfo {
code: code.into(),
message: self.to_string(),
}
}
}
/// Cheap-to-clone handle.
#[derive(Debug, Clone)]
pub struct SessionManager {
inner: Arc<Inner>,
}
#[derive(Debug)]
struct Inner {
daemon_id: String,
tmux: Tmux,
sidecar: SidecarStore,
worktree_mgr: WorktreeManager,
fifo_dir: PathBuf,
sessions: Mutex<HashMap<Uuid, SessionRecord>>,
events: broadcast::Sender<SessionEvent>,
}
impl SessionManager {
pub fn new(
daemon_id: impl Into<String>,
tmux: Tmux,
sidecar: SidecarStore,
worktree_mgr: WorktreeManager,
fifo_dir: PathBuf,
) -> Self {
let (events, _) = broadcast::channel(EVENTS_CAPACITY);
Self {
inner: Arc::new(Inner {
daemon_id: daemon_id.into(),
tmux,
sidecar,
worktree_mgr,
fifo_dir,
sessions: Mutex::new(HashMap::new()),
events,
}),
}
}
pub fn daemon_id(&self) -> &str {
&self.inner.daemon_id
}
pub fn events(&self) -> broadcast::Receiver<SessionEvent> {
self.inner.events.subscribe()
}
/// Snapshot of every live session. Suitable for `SessionsSnapshot`.
pub async fn snapshot(&self) -> Vec<Session> {
let sessions = self.inner.sessions.lock().await;
sessions.values().map(|r| r.session.clone()).collect()
}
/// Subscribe to a session's PTY broadcast. `None` if the session is
/// unknown to this daemon.
pub async fn subscribe_pty(&self, id: Uuid) -> Option<broadcast::Receiver<Vec<u8>>> {
let sessions = self.inner.sessions.lock().await;
sessions.get(&id).map(|r| r.bridge.subscribe())
}
pub async fn has(&self, id: Uuid) -> bool {
self.inner.sessions.lock().await.contains_key(&id)
}
/// Spawn a new session. On success returns the freshly-created `Session`.
pub async fn create(&self, req: CreateSessionRequest) -> Result<Session, SessionError> {
if req.daemon_id != self.inner.daemon_id {
return Err(SessionError::WrongDaemon {
wanted: req.daemon_id,
ours: self.inner.daemon_id.clone(),
});
}
let (effective_cwd, worktree_path, worktree_base_repo) = match &req.worktree {
Some(WorktreeSpec {
base_repo, branch, ..
}) => {
let path = self
.inner
.worktree_mgr
.add(base_repo, branch)
.map_err(SessionError::Worktree)?;
(path.clone(), Some(path), Some(base_repo.clone()))
}
None => (req.cwd.clone(), None, None),
};
let cleanup_worktree_on_kill = req
.worktree
.as_ref()
.map(|w| w.cleanup_on_kill)
.unwrap_or(false);
let id = Uuid::new_v4();
self.inner
.tmux
.new_session(id, &effective_cwd, &req.command)
.map_err(SessionError::Tmux)?;
let fifo_path = self.inner.fifo_dir.join(format!("{id}.fifo"));
let bridge = PtyBridge::start(fifo_path.clone()).await?;
self.inner
.tmux
.pipe_pane(id, &pipe_pane_command(&fifo_path))
.map_err(SessionError::Tmux)?;
let session = Session {
id,
daemon_id: self.inner.daemon_id.clone(),
cwd: effective_cwd,
command: req.command,
env: req.env,
metadata: req.metadata,
status: SessionStatus::Running,
created_at: Utc::now(),
};
self.inner
.sidecar
.write(&session)
.map_err(SessionError::Sidecar)?;
{
let mut sessions = self.inner.sessions.lock().await;
sessions.insert(
id,
SessionRecord {
session: session.clone(),
fifo_path,
worktree_path,
worktree_base_repo,
cleanup_worktree_on_kill,
bridge,
},
);
}
let _ = self.inner.events.send(SessionEvent::Added(session.clone()));
Ok(session)
}
/// Kill a session and remove its on-disk artifacts. The worktree is only
/// pruned if the create request set `cleanup_on_kill: true`.
pub async fn kill(&self, id: Uuid) -> Result<(), SessionError> {
let removed = {
let mut sessions = self.inner.sessions.lock().await;
sessions.remove(&id)
};
let record = removed.ok_or(SessionError::NotFound(id))?;
let _ = self.inner.tmux.kill_session(id);
let _ = std::fs::remove_file(&record.fifo_path);
self.inner
.sidecar
.delete(id)
.map_err(SessionError::Sidecar)?;
if record.cleanup_worktree_on_kill {
if let (Some(base), Some(path)) = (&record.worktree_base_repo, &record.worktree_path) {
let _ = self.inner.worktree_mgr.remove(base, path);
}
}
let _ = self.inner.events.send(SessionEvent::Removed(id));
Ok(())
}
/// Spawn a background task that polls tmux every `interval` and reaps any
/// in-memory sessions whose underlying tmux session has disappeared (the
/// command exited, the user ran `tmux kill-session`, etc.).
///
/// The task holds a `Weak<Inner>` and exits cleanly once the last
/// `SessionManager` clone is dropped — call this once from the daemon
/// startup path after [`Self::recover`] has populated the map.
pub fn start_liveness_watcher(&self, interval: Duration) {
let weak = Arc::downgrade(&self.inner);
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
// first tick fires immediately — skip it so we don't reap before
// the caller has a chance to create anything.
ticker.tick().await;
loop {
ticker.tick().await;
let Some(inner) = weak.upgrade() else {
return;
};
run_liveness_pass(&inner).await;
}
});
}
/// Re-attach to tmux sessions left behind by a previous daemon process.
///
/// For each `maestro-<uuid>` session we read its sidecar (or synthesise
/// minimal metadata if missing), re-issue `pipe-pane` to a fresh FIFO,
/// and start a `PtyBridge`. Sessions whose sidecar references a `cwd`
/// that no longer exists are still adopted — the spec does not require
/// us to repair them.
pub async fn recover(&self) -> Result<usize, SessionError> {
tokio::fs::create_dir_all(&self.inner.fifo_dir).await?;
let live = self.inner.tmux.list_sessions()?;
let mut adopted = 0usize;
for id in live {
let session = match self.inner.sidecar.read(id) {
Ok(Some(s)) => s,
_ => Session {
id,
daemon_id: self.inner.daemon_id.clone(),
cwd: PathBuf::from("/"),
command: String::new(),
env: Default::default(),
metadata: Default::default(),
status: SessionStatus::Running,
created_at: Utc::now(),
},
};
let fifo_path = self.inner.fifo_dir.join(format!("{id}.fifo"));
let bridge = PtyBridge::start(fifo_path.clone()).await?;
// Best-effort: re-pipe so output starts flowing again.
let _ = self
.inner
.tmux
.pipe_pane(id, &pipe_pane_command(&fifo_path));
let mut sessions = self.inner.sessions.lock().await;
sessions.insert(
id,
SessionRecord {
session: session.clone(),
fifo_path,
worktree_path: None,
worktree_base_repo: None,
cleanup_worktree_on_kill: false,
bridge,
},
);
let _ = self.inner.events.send(SessionEvent::Added(session));
adopted += 1;
}
Ok(adopted)
}
pub async fn send_keys(&self, id: Uuid, bytes: &[u8]) -> Result<(), SessionError> {
if !self.has(id).await {
return Err(SessionError::NotFound(id));
}
self.inner
.tmux
.send_keys(id, bytes)
.map_err(SessionError::Tmux)
}
pub async fn resize(&self, id: Uuid, cols: u16, rows: u16) -> Result<(), SessionError> {
if !self.has(id).await {
return Err(SessionError::NotFound(id));
}
self.inner
.tmux
.resize_window(id, cols, rows)
.map_err(SessionError::Tmux)
}
}
/// One pass of the liveness watcher: diff the in-memory session set against
/// the tmux-reported one, and reap anything we still track but tmux has lost.
async fn run_liveness_pass(inner: &Arc<Inner>) {
// Snapshot the in-memory ids first; if we own no sessions there is
// nothing to reap and no reason to spawn a tmux subprocess (this also
// avoids racing with the very first `new-session` that would otherwise
// be competing to start the tmux server).
let known: Vec<Uuid> = {
let sessions = inner.sessions.lock().await;
if sessions.is_empty() {
return;
}
sessions.keys().copied().collect()
};
let live_ids = match inner.tmux.list_sessions() {
Ok(ids) => ids,
Err(e) => {
tracing::warn!(error = ?e, "liveness watcher: tmux list-sessions failed");
return;
}
};
let live: HashSet<Uuid> = live_ids.into_iter().collect();
for id in known {
if !live.contains(&id) {
reap_dead_session(inner, id).await;
}
}
}
/// Remove a session whose tmux session is already gone: drop the in-memory
/// record, delete the sidecar + FIFO, clean up its worktree if requested, and
/// fire `SessionEvent::Removed`. No tmux kill — there is nothing left to kill.
async fn reap_dead_session(inner: &Arc<Inner>, id: Uuid) {
let record = {
let mut sessions = inner.sessions.lock().await;
sessions.remove(&id)
};
let Some(record) = record else {
return;
};
tracing::info!(%id, "session vanished from tmux — reaping");
let _ = std::fs::remove_file(&record.fifo_path);
if let Err(e) = inner.sidecar.delete(id) {
tracing::warn!(error = ?e, %id, "reap: failed to delete sidecar");
}
if record.cleanup_worktree_on_kill {
if let (Some(base), Some(path)) = (&record.worktree_base_repo, &record.worktree_path) {
let _ = inner.worktree_mgr.remove(base, path);
}
}
let _ = inner.events.send(SessionEvent::Removed(id));
}