Compare commits

..
5 changed files with 259 additions and 41 deletions
+28
View File
@@ -158,6 +158,7 @@ enum Command {
enum ChildCommand {
Signal(libc::c_int),
Reap(oneshot::Sender<bool>),
}
/// Why the handshake did not produce a session id.
@@ -206,6 +207,18 @@ impl AcpSession {
self.exit.borrow().is_some()
}
/// Reap the child if it has already exited.
pub async fn reap_if_exited(&self) -> bool {
if self.has_exited() {
return true;
}
let (sent, received) = oneshot::channel();
if self.child_commands.send(ChildCommand::Reap(sent)).is_err() {
return self.has_exited();
}
received.await.unwrap_or_else(|_| self.has_exited())
}
/// Wait for the child to exit.
pub async fn wait(&self) -> Option<ExitStatus> {
wait_exit(&self.exit).await
@@ -392,6 +405,21 @@ async fn supervise_child(
// so its PID cannot be reused in that window.
unsafe { libc::kill(pid as libc::pid_t, sig) };
}
Some(ChildCommand::Reap(done)) => match child.try_wait() {
Ok(Some(status)) => {
let _ = done.send(true);
tracing::debug!(pid, ?status, "{ADAPTER_BIN} exited");
let _ = exit_tx.send(Some(status));
return;
}
Ok(None) => {
let _ = done.send(false);
}
Err(error) => {
tracing::warn!(pid, %error, "could not reap {ADAPTER_BIN}");
let _ = done.send(false);
}
},
None => {
// The session vanished without a command receiver. This
// task still owns the unreaped child, so this cannot hit
+69 -37
View File
@@ -379,8 +379,26 @@ impl SessionManager {
if children.is_empty() {
return;
}
tracing::info!(count = children.len(), "shutdown: ending adapter children");
let terminations = children
let reaped =
futures_util::future::join_all(children.iter().map(|(_, acp, _)| acp.reap_if_exited()))
.await;
let mut natural_exits = Vec::new();
let mut live_children = Vec::new();
for (child, exited) in children.iter().cloned().zip(reaped) {
if exited {
natural_exits.push(child.0);
} else {
live_children.push(child);
}
}
for id in natural_exits {
mark_session_dead_with_worktree_settlement(&self.inner, id, true).await;
}
tracing::info!(
count = live_children.len(),
"shutdown: ending adapter children"
);
let terminations = live_children
.iter()
.map(|(_, acp, _)| acp.terminate(maestro_acp::CANCEL_GRACE, maestro_acp::TERM_GRACE));
futures_util::future::join_all(terminations).await;
@@ -389,7 +407,7 @@ impl SessionManager {
.filter_map(|(_, _, done)| done.clone())
.map(wait_for_event_pump);
futures_util::future::join_all(drains).await;
for (id, _, _) in children {
for (id, _, _) in live_children {
mark_session_dead_after_shutdown(&self.inner, id).await;
}
}
@@ -930,9 +948,9 @@ impl SessionManager {
adopted += 1;
}
// Structured sessions have no tmux process to discover. Their
// sidecars are the recovery inventory: when an ACP id exists, start a
// fresh adapter process and load that conversation into it.
// Structured sessions have no tmux process to discover. A durable
// marker written during graceful shutdown is the only evidence that a
// sidecar may be resumed; older and naturally-dead sidecars stay dead.
for mut session in self
.inner
.sidecar
@@ -945,6 +963,7 @@ impl SessionManager {
}
let id = session.id;
let recoverable = self.inner.sidecar.take_recovery_marker(id)?;
let mut session_env = sanitize_env(&session.env)?;
let fifo_path = self.inner.fifo_dir.join(format!("{id}.fifo"));
let bridge =
@@ -960,40 +979,50 @@ impl SessionManager {
session_env.insert("MAESTRO_URL".into(), url);
}
let (acp, failure) = match session.acp_session_id.clone() {
Some(acp_session_id) => {
let config = SpawnConfig::new(
&session.cwd,
session_env
.iter()
.map(|(key, value)| (key.clone(), value.clone()))
.collect(),
)
.load_session(acp_session_id);
match maestro_acp::spawn(config).await {
Ok(acp) => (Some(Arc::new(acp)), None),
Err(error) => {
tracing::warn!(%id, %error, "structured session: adapter failed to resume");
(None, Some(error.to_string()))
let (acp, failure) = if recoverable {
match session.acp_session_id.clone() {
Some(acp_session_id) => {
let config = SpawnConfig::new(
&session.cwd,
session_env
.iter()
.map(|(key, value)| (key.clone(), value.clone()))
.collect(),
)
.load_session(acp_session_id);
match maestro_acp::spawn(config).await {
Ok(acp) => (Some(Arc::new(acp)), None),
Err(error) => {
tracing::warn!(%id, %error, "structured session: adapter failed to resume");
(None, Some(error.to_string()))
}
}
}
None => (
None,
Some("structured session has no stored ACP session id".into()),
),
}
None => (
None,
Some("structured session has no stored ACP session id".into()),
),
} else {
(None, None)
};
session.status = if failure.is_some() {
session.status = if recoverable {
if failure.is_some() {
SessionStatus::Dead
} else {
SessionStatus::Running
}
} else {
SessionStatus::Dead
} else {
SessionStatus::Running
};
session.activity = if failure.is_some() {
ActivityState::Error
} else {
ActivityState::Active
};
session.activity_detail = failure;
if recoverable {
session.activity = if failure.is_some() {
ActivityState::Error
} else {
ActivityState::Active
};
session.activity_detail = failure;
}
let acp_events = if let Some(acp) = &acp {
match self.inner.sidecar.open_events(id) {
@@ -1723,14 +1752,16 @@ async fn mark_session_dead(inner: &Arc<Inner>, id: Uuid) {
/// A daemon restart terminates ACP children, but it is not a natural adapter
/// exit: the sidecar is the recovery inventory, including its worktree.
async fn mark_session_dead_after_shutdown(inner: &Arc<Inner>, id: Uuid) {
mark_session_dead_with_worktree_settlement(inner, id, false).await;
if mark_session_dead_with_worktree_settlement(inner, id, false).await {
let _ = inner.sidecar.mark_for_recovery(id);
}
}
async fn mark_session_dead_with_worktree_settlement(
inner: &Arc<Inner>,
id: Uuid,
settle_worktree: bool,
) {
) -> bool {
let flipped = {
let mut sessions = inner.sessions.lock().await;
match sessions.get_mut(&id) {
@@ -1750,7 +1781,7 @@ async fn mark_session_dead_with_worktree_settlement(
}
};
let Some((mut session, worktree)) = flipped else {
return;
return false;
};
tracing::info!(%id, "session process gone — marking dead");
@@ -1781,6 +1812,7 @@ async fn mark_session_dead_with_worktree_settlement(
let _ = inner.sidecar.write(&session);
let _ = inner.events.send(CatalogEvent::SessionUpdated { session });
true
}
/// Apply the exit rule (#82) to a vanished session's worktree: probe what
+25 -2
View File
@@ -56,6 +56,28 @@ impl SidecarStore {
self.dir.join(format!("{id}.events.jsonl"))
}
/// Durable evidence that graceful daemon shutdown ended this structured
/// session and the next daemon may resume it.
pub fn recovery_path_for(&self, id: Uuid) -> PathBuf {
self.dir.join(format!("{id}.recover"))
}
/// Mark `id` as eligible for one structured-session recovery attempt.
pub fn mark_for_recovery(&self, id: Uuid) -> Result<(), SidecarError> {
std::fs::create_dir_all(&self.dir)?;
std::fs::write(self.recovery_path_for(id), [])?;
Ok(())
}
/// Consume the graceful-shutdown recovery marker for `id`.
pub fn take_recovery_marker(&self, id: Uuid) -> Result<bool, SidecarError> {
match std::fs::remove_file(self.recovery_path_for(id)) {
Ok(()) => Ok(true),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(error) => Err(error.into()),
}
}
/// Open the event log for `id` for appending, creating it (and the
/// directory) when missing. One handle per live session: the caller
/// keeps it for the session's lifetime rather than reopening per
@@ -164,10 +186,11 @@ impl SidecarStore {
}
}
/// Delete the sidecar and the event log for `id`. No-op for whichever
/// does not exist.
/// Delete the sidecar, event log, and recovery marker for `id`. No-op for
/// whichever does not exist.
pub fn delete(&self, id: Uuid) -> Result<(), SidecarError> {
remove_if_present(&self.events_path_for(id))?;
remove_if_present(&self.recovery_path_for(id))?;
remove_if_present(&self.path_for(id))
}
+122 -2
View File
@@ -232,6 +232,12 @@ async fn recovery_loads_the_acp_session_without_duplicating_history() {
let old_pid = fake.pid();
fx.manager.shutdown().await;
wait_for_exit(old_pid).await;
assert!(
fx.sidecar_dir()
.join(format!("{}.recover", session.id))
.exists(),
"graceful shutdown must record recovery provenance"
);
let revived = SessionManager::new(
Arc::new(fx.tmux.tmux.clone()),
@@ -498,6 +504,100 @@ async fn shutdown_reaps_every_adapter_child() {
assert_eq!(sidecar.status, SessionStatus::Dead);
}
#[tokio::test]
async fn dismissal_during_shutdown_does_not_restore_recovery_artifacts() {
let fx = DaemonFixture::new();
let dismissed = Fake::new();
let blocker = Fake::new();
let cwd = TempDir::new().unwrap();
let mut env = dismissed.env();
env.insert("FAKE_ACP_EXIT_ON_CANCEL".into(), "1".into());
let session = fx
.manager
.create(structured(cwd.path(), env))
.await
.expect("create");
fx.manager
.create(structured(cwd.path(), blocker.env()))
.await
.expect("create blocker");
let manager = fx.manager.clone();
let shutdown = tokio::spawn(async move { manager.shutdown().await });
wait_for_async(Duration::from_secs(5), || async {
dismissed.log().contains("recv session/cancel")
})
.await;
fx.manager.kill(session.id, false).await.expect("dismiss");
assert!(
!fx.sidecar_dir()
.join(format!("{}.json", session.id))
.exists(),
"dismissal must finish before shutdown settles its stale snapshot"
);
shutdown.await.expect("shutdown task");
assert!(!fx.manager.has(session.id).await);
assert!(
!fx.sidecar_dir()
.join(format!("{}.recover", session.id))
.exists(),
"dismissal must leave no recovery marker"
);
assert!(
!fx.sidecar_dir()
.join(format!("{}.json", session.id))
.exists(),
"dismissal must leave no sidecar"
);
assert!(
!fx.sidecar_dir()
.join(format!("{}.events.jsonl", session.id))
.exists(),
"dismissal must leave no event log"
);
}
#[tokio::test]
async fn shutdown_does_not_recover_an_adapter_that_already_exited() {
let fx = DaemonFixture::new();
let fake = Fake::new();
let cwd = TempDir::new().unwrap();
let mut env = fake.env();
env.insert("FAKE_ACP_EXIT_AFTER_NEW".into(), "1".into());
let session = fx
.manager
.create(structured(cwd.path(), env))
.await
.expect("create");
wait_for_exit(fake.pid()).await;
fx.manager.shutdown().await;
assert!(
!fx.sidecar_dir()
.join(format!("{}.recover", session.id))
.exists(),
"a naturally exited adapter is not recoverable"
);
let revived = SessionManager::new(
Arc::new(fx.tmux.tmux.clone()),
SidecarStore::new(fx.sidecar_dir()),
WorktreeManager::new(None),
fx.fifo_dir(),
);
assert_eq!(revived.recover().await.expect("recover"), 1);
assert_eq!(revived.status(session.id).await, Some(SessionStatus::Dead));
assert!(
!fake.log().contains("recv session/load"),
"a naturally dead adapter must not be resumed: {}",
fake.log()
);
}
#[tokio::test]
async fn shutdown_waits_for_structured_creation_and_rejects_new_creates() {
let fx = DaemonFixture::new();
@@ -600,7 +700,27 @@ async fn adapter_exit_marks_the_session_dead_and_keeps_it_listed() {
})
.await;
assert!(fx.manager.has(session.id).await);
assert!(
!fx.sidecar_dir()
.join(format!("{}.recover", session.id))
.exists(),
"a naturally exited adapter is never marked for recovery"
);
fx.manager.kill(session.id, false).await.expect("dismiss");
assert!(!fx.manager.has(session.id).await);
let revived = SessionManager::new(
Arc::new(fx.tmux.tmux.clone()),
SidecarStore::new(fx.sidecar_dir()),
WorktreeManager::new(None),
fx.fifo_dir(),
);
assert_eq!(revived.recover().await.expect("recover"), 1);
assert_eq!(revived.status(session.id).await, Some(SessionStatus::Dead));
assert!(
!fake.log().contains("recv session/load"),
"a naturally dead adapter must not be resumed: {}",
fake.log()
);
revived.kill(session.id, false).await.expect("dismiss");
assert!(!revived.has(session.id).await);
}
+15
View File
@@ -68,15 +68,30 @@ fn delete_removes_file_and_is_idempotent() {
let s = sample(id);
store.write(&s).unwrap();
store.mark_for_recovery(id).unwrap();
assert!(store.path_for(id).exists());
assert!(store.recovery_path_for(id).exists());
store.delete(id).unwrap();
assert!(!store.path_for(id).exists());
assert!(!store.recovery_path_for(id).exists());
// second delete is a no-op
store.delete(id).unwrap();
}
#[test]
fn recovery_marker_is_consumed_once() {
let dir = TempDir::new().unwrap();
let store = SidecarStore::new(dir.path());
let id = Uuid::new_v4();
assert!(!store.take_recovery_marker(id).unwrap());
store.mark_for_recovery(id).unwrap();
assert!(store.take_recovery_marker(id).unwrap());
assert!(!store.take_recovery_marker(id).unwrap());
}
#[test]
fn load_all_recovers_every_sidecar_and_skips_garbage() {
let dir = TempDir::new().unwrap();