Serialize replay reads with event appends, acquire the live receiver under that lock, and drain the event pump during shutdown. Recovery now removes its FIFO when opening the log fails.
491 lines
16 KiB
Rust
491 lines
16 KiB
Rust
//! Structured sessions (#134): a `claude-code-acp` child per session instead
|
|
//! of a tmux pane. Runs against the fake adapter in
|
|
//! `crates/maestro-acp/tests/fake_acp_agent.py`; nothing here needs Claude.
|
|
|
|
mod common;
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use common::{wait_for_async, DaemonFixture};
|
|
use maestro_acp::schema::{ContentBlock, SessionUpdate};
|
|
use maestro_acp::{ADAPTER_BIN, INSTALL_COMMAND};
|
|
use maestro_daemon::sidecar::{EventCursor, SidecarStore};
|
|
use maestro_daemon::worktree::WorktreeManager;
|
|
use maestro_daemon::SessionManager;
|
|
use maestro_protocol::{ActivityState, CreateSessionRequest, SessionStatus, SessionView};
|
|
use tempfile::TempDir;
|
|
|
|
fn fake_agent() -> PathBuf {
|
|
Path::new(env!("CARGO_MANIFEST_DIR")).join("../maestro-acp/tests/fake_acp_agent.py")
|
|
}
|
|
|
|
/// The fake adapter installed as `claude-code-acp` in a temp `bin/`, with
|
|
/// the log file it appends to. `env()` is what a spawn request carries.
|
|
struct Fake {
|
|
dir: TempDir,
|
|
}
|
|
|
|
impl Fake {
|
|
fn new() -> Self {
|
|
let dir = TempDir::new().unwrap();
|
|
let bin = dir.path().join("bin");
|
|
std::fs::create_dir_all(&bin).unwrap();
|
|
std::fs::copy(fake_agent(), bin.join(ADAPTER_BIN)).unwrap();
|
|
Self { dir }
|
|
}
|
|
|
|
fn log_path(&self) -> PathBuf {
|
|
self.dir.path().join("agent.log")
|
|
}
|
|
|
|
fn log(&self) -> String {
|
|
std::fs::read_to_string(self.log_path()).unwrap_or_default()
|
|
}
|
|
|
|
/// Fake `bin/` first so it shadows any real adapter, then the process
|
|
/// `PATH` so the script's `python3` still resolves.
|
|
fn env(&self) -> HashMap<String, String> {
|
|
let path = format!(
|
|
"{}:{}",
|
|
self.dir.path().join("bin").display(),
|
|
std::env::var("PATH").unwrap_or_default()
|
|
);
|
|
HashMap::from([
|
|
("PATH".to_string(), path),
|
|
(
|
|
"FAKE_ACP_LOG".to_string(),
|
|
self.log_path().to_string_lossy().into_owned(),
|
|
),
|
|
])
|
|
}
|
|
|
|
/// The pid the fake logged at start, once it has.
|
|
fn pid(&self) -> u32 {
|
|
self.log()
|
|
.lines()
|
|
.filter_map(|line| {
|
|
line.strip_prefix("start pid=")?
|
|
.split(' ')
|
|
.next()?
|
|
.parse()
|
|
.ok()
|
|
})
|
|
.next_back()
|
|
.expect("pid in log")
|
|
}
|
|
}
|
|
|
|
fn event_text(event: &maestro_daemon::sidecar::LoggedEvent) -> &str {
|
|
match &event.notification.update {
|
|
SessionUpdate::AgentMessageChunk(chunk) => match &chunk.content {
|
|
ContentBlock::Text(text) => &text.text,
|
|
other => panic!("unexpected content {other:?}"),
|
|
},
|
|
other => panic!("unexpected update {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn structured(cwd: &Path, env: HashMap<String, String>) -> CreateSessionRequest {
|
|
CreateSessionRequest {
|
|
view: SessionView::Structured,
|
|
cwd: cwd.to_path_buf(),
|
|
command: "claude".into(),
|
|
env,
|
|
metadata: HashMap::new(),
|
|
worktree: None,
|
|
}
|
|
}
|
|
|
|
fn pid_alive(pid: u32) -> bool {
|
|
std::fs::read_to_string(format!("/proc/{pid}/stat"))
|
|
.map(|s| !s.contains(") Z "))
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
async fn wait_for_exit(pid: u32) {
|
|
let deadline = Instant::now() + Duration::from_secs(5);
|
|
while pid_alive(pid) && Instant::now() < deadline {
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
assert!(!pid_alive(pid), "pid {pid} still alive");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn structured_spawn_starts_the_adapter_in_cwd_with_session_env() {
|
|
let fx = DaemonFixture::new();
|
|
let fake = Fake::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
let mut env = fake.env();
|
|
env.insert("FAKE_ACP_ECHO_ENV".into(), "MAESTRO_SESSION_ID".into());
|
|
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), env))
|
|
.await
|
|
.expect("create");
|
|
|
|
assert_eq!(session.view, SessionView::Structured);
|
|
assert_eq!(session.status, SessionStatus::Running);
|
|
let acp_id = session.acp_session_id.clone().expect("acp session id");
|
|
assert!(acp_id.starts_with("fake-"), "{acp_id}");
|
|
|
|
// The child ran in the session cwd with the session's env — including
|
|
// the identity the daemon injects, so the agent knows which session it
|
|
// is (#103).
|
|
let log = fake.log();
|
|
let cwd_canon = cwd.path().canonicalize().unwrap();
|
|
assert!(
|
|
log.contains(&format!("cwd={}", cwd_canon.display())),
|
|
"{log}"
|
|
);
|
|
assert!(
|
|
log.contains(&format!("env MAESTRO_SESSION_ID={}", session.id)),
|
|
"{log}"
|
|
);
|
|
|
|
// No tmux pane was created for it: the agent is the daemon's child.
|
|
let panes = fx.tmux.tmux.list_sessions().expect("list");
|
|
assert!(
|
|
!panes.contains(&session.id),
|
|
"structured session got a tmux pane"
|
|
);
|
|
|
|
// The sidecar carries the id a respawn will pass to session/load (#135).
|
|
let sidecar: maestro_protocol::Session = serde_json::from_slice(
|
|
&std::fs::read(fx.sidecar_dir().join(format!("{}.json", session.id))).unwrap(),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(sidecar.acp_session_id.as_deref(), Some(acp_id.as_str()));
|
|
|
|
wait_for_async(Duration::from_secs(5), || async {
|
|
fx.manager
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.is_ok_and(|events| !events.is_empty())
|
|
})
|
|
.await;
|
|
|
|
// Live delivery happens only after the notification is durable.
|
|
let (replay, mut updates) = fx
|
|
.manager
|
|
.subscribe_acp_events_with_replay(session.id, EventCursor::Since(0), 10)
|
|
.await
|
|
.expect("replay read")
|
|
.expect("event subscription");
|
|
assert_eq!(replay.len(), 1);
|
|
assert!(updates.try_recv().is_err(), "replay duplicated onto live");
|
|
unsafe {
|
|
libc::kill(fake.pid() as libc::pid_t, libc::SIGUSR1);
|
|
}
|
|
let live = tokio::time::timeout(Duration::from_secs(5), updates.recv())
|
|
.await
|
|
.expect("an update within 5s")
|
|
.expect("broadcast open");
|
|
assert_eq!(event_text(&live), "tick 1");
|
|
let stored = fx
|
|
.manager
|
|
.read_acp_events(session.id, EventCursor::Since(live.seq - 1), 1)
|
|
.expect("read event logged before broadcast");
|
|
assert_eq!(stored, vec![live]);
|
|
|
|
fx.manager.kill(session.id, false).await.expect("kill");
|
|
assert!(!fx
|
|
.sidecar_dir()
|
|
.join(format!("{}.events.jsonl", session.id))
|
|
.exists());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn recovery_loads_the_acp_session_without_duplicating_history() {
|
|
let fx = DaemonFixture::new();
|
|
let fake = Fake::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
let mut env = fake.env();
|
|
env.insert("FAKE_ACP_LOAD_REPLAY".into(), "2".into());
|
|
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), env))
|
|
.await
|
|
.expect("create");
|
|
let acp_id = session.acp_session_id.clone().expect("acp id");
|
|
wait_for_async(Duration::from_secs(5), || async {
|
|
fx.manager
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.is_ok_and(|events| events.len() == 1)
|
|
})
|
|
.await;
|
|
unsafe {
|
|
libc::kill(fake.pid() as libc::pid_t, libc::SIGUSR1);
|
|
}
|
|
wait_for_async(Duration::from_secs(5), || async {
|
|
fx.manager
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.is_ok_and(|events| events.len() == 2)
|
|
})
|
|
.await;
|
|
|
|
let old_pid = fake.pid();
|
|
fx.manager.shutdown().await;
|
|
wait_for_exit(old_pid).await;
|
|
|
|
let revived = SessionManager::new(
|
|
Arc::new(fx.tmux.tmux.clone()),
|
|
SidecarStore::new(fx.sidecar_dir()),
|
|
WorktreeManager::new(None),
|
|
fx.fifo_dir(),
|
|
);
|
|
assert_eq!(revived.recover().await.expect("recover"), 1);
|
|
assert_eq!(
|
|
revived.status(session.id).await,
|
|
Some(SessionStatus::Running)
|
|
);
|
|
let log = fake.log();
|
|
assert!(
|
|
log.contains(&format!("recv session/load sessionId={acp_id}")),
|
|
"{log}"
|
|
);
|
|
wait_for_async(Duration::from_secs(5), || async {
|
|
revived
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.is_ok_and(|events| events.len() == 3)
|
|
})
|
|
.await;
|
|
|
|
let (replay, mut live) = revived
|
|
.subscribe_acp_events_with_replay(session.id, EventCursor::Since(0), 10)
|
|
.await
|
|
.expect("replay read")
|
|
.expect("recovered event subscription");
|
|
assert_eq!(replay.len(), 3);
|
|
assert!(live.try_recv().is_err(), "replay duplicated onto live");
|
|
let new_pid = fake.pid();
|
|
assert_ne!(new_pid, old_pid);
|
|
unsafe {
|
|
libc::kill(new_pid as libc::pid_t, libc::SIGUSR1);
|
|
}
|
|
let event = tokio::time::timeout(Duration::from_secs(5), live.recv())
|
|
.await
|
|
.expect("live event after recovery")
|
|
.expect("broadcast open");
|
|
assert_eq!(event.seq, 4);
|
|
assert_eq!(event_text(&event), "tick 1");
|
|
|
|
let stored = revived
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.expect("recovered log");
|
|
assert_eq!(stored.len(), 4, "session/load replay must not be appended");
|
|
assert_eq!(
|
|
stored.iter().map(event_text).collect::<Vec<_>>(),
|
|
["hello from fake", "tick 1", "hello after load", "tick 1"]
|
|
);
|
|
|
|
revived.kill(session.id, false).await.expect("kill");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn failed_recovery_cleans_its_fifo() {
|
|
let fx = DaemonFixture::new();
|
|
let fake = Fake::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), fake.env()))
|
|
.await
|
|
.expect("create");
|
|
let old_pid = fake.pid();
|
|
fx.manager.shutdown().await;
|
|
wait_for_exit(old_pid).await;
|
|
|
|
let events_path = fx
|
|
.sidecar_dir()
|
|
.join(format!("{}.events.jsonl", session.id));
|
|
std::fs::remove_file(&events_path).unwrap();
|
|
std::fs::create_dir(&events_path).unwrap();
|
|
let revived = SessionManager::new(
|
|
Arc::new(fx.tmux.tmux.clone()),
|
|
SidecarStore::new(fx.sidecar_dir()),
|
|
WorktreeManager::new(None),
|
|
fx.fifo_dir(),
|
|
);
|
|
|
|
assert!(revived.recover().await.is_err());
|
|
assert!(!fx.fifo_dir().join(format!("{}.fifo", session.id)).exists());
|
|
let failed_pid = fake.pid();
|
|
assert_ne!(failed_pid, old_pid);
|
|
wait_for_exit(failed_pid).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn missing_adapter_yields_a_failed_session_naming_the_install_command() {
|
|
let fx = DaemonFixture::new();
|
|
let empty = TempDir::new().unwrap();
|
|
let cwd = TempDir::new().unwrap();
|
|
let env = HashMap::from([(
|
|
"PATH".to_string(),
|
|
empty.path().to_string_lossy().into_owned(),
|
|
)]);
|
|
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), env))
|
|
.await
|
|
.expect("a failed session, not an error");
|
|
|
|
assert_eq!(session.status, SessionStatus::Dead);
|
|
assert_eq!(session.activity, ActivityState::Error);
|
|
let detail = session.activity_detail.clone().expect("actionable detail");
|
|
assert!(detail.contains(ADAPTER_BIN), "{detail}");
|
|
assert!(detail.contains(INSTALL_COMMAND), "{detail}");
|
|
assert_eq!(session.acp_session_id, None);
|
|
|
|
// It is a real session: listed, on disk, and dismissable like any other
|
|
// dead one.
|
|
assert!(fx.manager.has(session.id).await);
|
|
assert!(fx
|
|
.sidecar_dir()
|
|
.join(format!("{}.json", session.id))
|
|
.exists());
|
|
fx.manager.kill(session.id, false).await.expect("dismiss");
|
|
assert!(!fx.manager.has(session.id).await);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn kill_sends_cancel_then_sigterm_and_removes_the_sidecar() {
|
|
let fx = DaemonFixture::new();
|
|
let fake = Fake::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
let mut events = fx.manager.events();
|
|
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), fake.env()))
|
|
.await
|
|
.expect("create");
|
|
let pid = fake.pid();
|
|
assert!(pid_alive(pid));
|
|
|
|
fx.manager.kill(session.id, false).await.expect("kill");
|
|
|
|
wait_for_exit(pid).await;
|
|
let log = fake.log();
|
|
let cancel = log.find("recv session/cancel").expect("cancel logged");
|
|
let term = log.find("signal SIGTERM").expect("SIGTERM logged");
|
|
assert!(cancel < term, "cancel must precede SIGTERM:\n{log}");
|
|
assert!(!fx.manager.has(session.id).await);
|
|
assert!(!fx
|
|
.sidecar_dir()
|
|
.join(format!("{}.json", session.id))
|
|
.exists());
|
|
|
|
let mut removed = false;
|
|
while let Ok(ev) = events.try_recv() {
|
|
if matches!(ev, maestro_protocol::CatalogEvent::SessionRemoved { id } if id == session.id) {
|
|
removed = true;
|
|
}
|
|
}
|
|
assert!(removed, "SessionRemoved never broadcast");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn shutdown_reaps_every_adapter_child() {
|
|
let fx = DaemonFixture::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
let a = Fake::new();
|
|
let b = Fake::new();
|
|
|
|
let sa = fx
|
|
.manager
|
|
.create(structured(cwd.path(), a.env()))
|
|
.await
|
|
.expect("create a");
|
|
let sb = fx
|
|
.manager
|
|
.create(structured(cwd.path(), b.env()))
|
|
.await
|
|
.expect("create b");
|
|
let (pa, pb) = (a.pid(), b.pid());
|
|
assert!(pid_alive(pa) && pid_alive(pb));
|
|
|
|
fx.manager.shutdown().await;
|
|
|
|
wait_for_exit(pa).await;
|
|
wait_for_exit(pb).await;
|
|
assert!(a.log().contains("signal SIGTERM"), "{}", a.log());
|
|
assert!(b.log().contains("signal SIGTERM"), "{}", b.log());
|
|
// Still listed, now Dead: the sidecar is truthful for whoever reads it
|
|
// after the restart (#135).
|
|
assert_eq!(fx.manager.status(sa.id).await, Some(SessionStatus::Dead));
|
|
assert_eq!(fx.manager.status(sb.id).await, Some(SessionStatus::Dead));
|
|
let sidecar: maestro_protocol::Session = serde_json::from_slice(
|
|
&std::fs::read(fx.sidecar_dir().join(format!("{}.json", sa.id))).unwrap(),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(sidecar.status, SessionStatus::Dead);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn shutdown_drains_final_notifications_to_the_log() {
|
|
let fx = DaemonFixture::new();
|
|
let fake = Fake::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
let mut env = fake.env();
|
|
env.insert("FAKE_ACP_EMIT_ON_CANCEL".into(), "1".into());
|
|
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), env))
|
|
.await
|
|
.expect("create");
|
|
wait_for_async(Duration::from_secs(5), || async {
|
|
fx.manager
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.is_ok_and(|events| events.len() == 1)
|
|
})
|
|
.await;
|
|
|
|
fx.manager.shutdown().await;
|
|
|
|
let stored = fx
|
|
.manager
|
|
.read_acp_events(session.id, EventCursor::Since(0), 10)
|
|
.expect("event log after shutdown");
|
|
assert_eq!(
|
|
stored.iter().map(event_text).collect::<Vec<_>>(),
|
|
["hello from fake", "final on cancel"]
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn adapter_exit_marks_the_session_dead_and_keeps_it_listed() {
|
|
let fx = DaemonFixture::new();
|
|
let fake = Fake::new();
|
|
let cwd = TempDir::new().unwrap();
|
|
|
|
let session = fx
|
|
.manager
|
|
.create(structured(cwd.path(), fake.env()))
|
|
.await
|
|
.expect("create");
|
|
fx.manager.start_liveness_watcher(Duration::from_millis(50));
|
|
|
|
// The agent dies out from under the daemon: the liveness pass, not tmux,
|
|
// is what notices for a structured session.
|
|
let pid = fake.pid();
|
|
unsafe {
|
|
libc::kill(pid as libc::pid_t, libc::SIGTERM);
|
|
}
|
|
wait_for_exit(pid).await;
|
|
|
|
wait_for_async(Duration::from_secs(3), || async {
|
|
fx.manager.status(session.id).await == Some(SessionStatus::Dead)
|
|
})
|
|
.await;
|
|
assert!(fx.manager.has(session.id).await);
|
|
|
|
fx.manager.kill(session.id, false).await.expect("dismiss");
|
|
assert!(!fx.manager.has(session.id).await);
|
|
}
|