//! Cross-process end-to-end tests for the headless Maestro slice. //! //! Each test brings up a real `maestro-registry` plus 1-2 real `maestrod` //! subprocesses, isolates them in temp dirs, and drives them through the //! public HTTP+WS surface using the same client APIs a real CLI/web client //! would. //! //! These tests require `tmux` and `git` on `$PATH`. #![allow(clippy::result_large_err)] use std::collections::HashMap; use std::process::Command; use std::time::{Duration, Instant}; use futures_util::{SinkExt, StreamExt}; use maestro_e2e::util::wait_for_async; use maestro_e2e::{Daemon, DaemonOptions, Registry, TestClient}; use maestro_protocol::{CreateSessionRequest, WorktreeSpec}; use tempfile::TempDir; use tokio_tungstenite::tungstenite::Message as WsMessage; const TOKEN: &str = "e2e-shared-token"; async fn bring_up(daemon_ids: &[&str]) -> (Registry, Vec, TestClient) { let registry = Registry::spawn(TOKEN).await; let mut daemons = Vec::with_capacity(daemon_ids.len()); for id in daemon_ids { let d = Daemon::spawn(id, ®istry.http_base(), TOKEN).await; daemons.push(d); } let client = TestClient::new(®istry.http_base(), ®istry.ws_base()); client.bootstrap().await.expect("bootstrap"); // Wait until both daemons have connected and announced themselves. let expected: Vec = daemon_ids.iter().map(|s| s.to_string()).collect(); wait_for_async("daemons to register", Duration::from_secs(10), || { let client = client.clone(); let expected = expected.clone(); async move { match client.list_daemons().await { Ok(ds) => expected .iter() .all(|id| ds.iter().any(|d| &d.daemon_id == id)), Err(_) => false, } } }) .await; (registry, daemons, client) } fn create_req(daemon_id: &str, cwd: &std::path::Path, command: &str) -> CreateSessionRequest { CreateSessionRequest { daemon_id: daemon_id.into(), cwd: cwd.to_path_buf(), command: command.into(), env: HashMap::new(), metadata: HashMap::new(), worktree: None, } } #[tokio::test(flavor = "multi_thread")] async fn lists_sessions_from_both_daemons() { let (_reg, _daemons, client) = bring_up(&["alpha", "beta"]).await; let cwd = TempDir::new().unwrap(); let a = client .spawn_session(&create_req("alpha", cwd.path(), "sleep 30")) .await .expect("spawn alpha"); let b = client .spawn_session(&create_req("beta", cwd.path(), "sleep 30")) .await .expect("spawn beta"); let sessions = client.list_sessions().await.expect("list"); assert!(sessions .iter() .any(|s| s.id == a.id && s.daemon_id == "alpha")); assert!(sessions .iter() .any(|s| s.id == b.id && s.daemon_id == "beta")); client.kill_session(a.id).await.expect("kill a"); client.kill_session(b.id).await.expect("kill b"); } #[tokio::test(flavor = "multi_thread")] async fn daemon_local_api_mirrors_peer_catalog() { // NAP-246: a daemon's GET /api/daemons + /api/sessions should reflect // peer state (not just self), because the registry fans out catalog // events to every connected daemon. let (_reg, daemons, client) = bring_up(&["alpha", "beta"]).await; let cwd = TempDir::new().unwrap(); // Create a session on beta via the registry path. let b = client .spawn_session(&create_req("beta", cwd.path(), "sleep 30")) .await .expect("spawn beta"); // Now query alpha's local HTTP API and confirm it sees beta + beta's // session. The mirror is eventually consistent, so poll. let alpha_addr = daemons[0].bind_addr; let alpha_base = format!("http://{alpha_addr}"); wait_for_async( "alpha mirrors beta in /api/daemons", Duration::from_secs(5), || { let base = alpha_base.clone(); async move { let resp = match reqwest::get(format!("{base}/api/daemons")).await { Ok(r) => r, Err(_) => return false, }; let list: Vec = match resp.json().await { Ok(v) => v, Err(_) => return false, }; list.iter().any(|d| d.daemon_id == "beta") } }, ) .await; wait_for_async( "alpha mirrors beta's session in /api/sessions", Duration::from_secs(5), || { let base = alpha_base.clone(); let want = b.id; async move { let resp = match reqwest::get(format!("{base}/api/sessions")).await { Ok(r) => r, Err(_) => return false, }; let list: Vec = match resp.json().await { Ok(v) => v, Err(_) => return false, }; list.iter().any(|s| s.id == want && s.daemon_id == "beta") } }, ) .await; // Both self and peer come back. let list: Vec = reqwest::get(format!("{alpha_base}/api/daemons")) .await .unwrap() .json() .await .unwrap(); let ids: Vec<&str> = list.iter().map(|d| d.daemon_id.as_str()).collect(); assert!(ids.contains(&"alpha")); assert!(ids.contains(&"beta")); client.kill_session(b.id).await.expect("kill b"); } #[tokio::test(flavor = "multi_thread")] async fn multi_attach_sees_same_output() { let (_reg, _daemons, client) = bring_up(&["solo"]).await; let cwd = TempDir::new().unwrap(); let s = client .spawn_session(&create_req("solo", cwd.path(), "bash --noprofile --norc")) .await .expect("spawn"); let info = client.connect_info(s.id).await.expect("connect-info"); let mut a = client.attach(&info, s.id).await.expect("attach a"); let mut b = client.attach(&info, s.id).await.expect("attach b"); // Drive input through client A; both attachments should observe it via // the daemon's PTY broadcast. let send = b"echo MAESTRO_E2E_SHARED\n".to_vec(); a.send(WsMessage::Binary(send)).await.expect("send keys"); let want = b"MAESTRO_E2E_SHARED"; let deadline = Instant::now() + Duration::from_secs(5); let mut buf_a = Vec::new(); let mut buf_b = Vec::new(); while Instant::now() < deadline { tokio::select! { msg = a.next() => match msg { Some(Ok(WsMessage::Binary(bytes))) => buf_a.extend(bytes), Some(Ok(_)) => {} _ => break, }, msg = b.next() => match msg { Some(Ok(WsMessage::Binary(bytes))) => buf_b.extend(bytes), Some(Ok(_)) => {} _ => break, }, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } if buf_a.windows(want.len()).any(|w| w == want) && buf_b.windows(want.len()).any(|w| w == want) { break; } } assert!( buf_a.windows(want.len()).any(|w| w == want), "client A never saw the echo" ); assert!( buf_b.windows(want.len()).any(|w| w == want), "client B never saw the echo" ); let _ = a.close(None).await; let _ = b.close(None).await; client.kill_session(s.id).await.expect("kill"); } #[tokio::test(flavor = "multi_thread")] async fn cleanup_on_kill_removes_worktree() { let (_reg, _daemons, client) = bring_up(&["host"]).await; let base = TempDir::new().unwrap(); init_repo(base.path()); let req = CreateSessionRequest { daemon_id: "host".into(), cwd: base.path().to_path_buf(), command: "sleep 30".into(), env: HashMap::new(), metadata: HashMap::new(), worktree: Some(WorktreeSpec { base_repo: base.path().to_path_buf(), branch: "feature/e2e".into(), cleanup_on_kill: true, }), }; let s = client.spawn_session(&req).await.expect("spawn worktree"); // The daemon's WorktreeManager puts new worktrees under // `/worktrees/`. We can't peek the daemon's state dir // directly from here, but we can verify by asking git: a worktree must // exist that lives outside the original repo. let worktrees = git_worktree_list(base.path()); assert!( worktrees.iter().any(|w| w.contains("feature/e2e")), "expected a worktree for feature/e2e branch, got: {worktrees:?}" ); client.kill_session(s.id).await.expect("kill"); // Give the daemon a moment to remove the worktree, then verify. let deadline = Instant::now() + Duration::from_secs(5); while Instant::now() < deadline { let after = git_worktree_list(base.path()); if !after.iter().any(|w| w.contains("feature/e2e")) { return; } tokio::time::sleep(Duration::from_millis(100)).await; } panic!( "worktree feature/e2e still present after kill: {:?}", git_worktree_list(base.path()) ); } #[tokio::test(flavor = "multi_thread")] async fn daemon_restart_recovers_session() { let (_reg, mut daemons, client) = bring_up(&["restarter"]).await; let cwd = TempDir::new().unwrap(); let s = client .spawn_session(&create_req("restarter", cwd.path(), "sleep 300")) .await .expect("spawn"); assert_eq!(s.daemon_id, "restarter"); // Stop the daemon and wait for the registry to notice the disconnect. // The catalog drops sessions for a daemon that's no longer connected. daemons[0].stop(); wait_for_async( "registry drops disconnected daemon", Duration::from_secs(10), || { let client = client.clone(); async move { client .list_daemons() .await .map(|ds| ds.iter().all(|d| d.daemon_id != "restarter")) .unwrap_or(false) } }, ) .await; // Bring the daemon back. It should recover its tmux session and the // registry catalog should repopulate it. daemons[0].restart().await; wait_for_async("restarter re-registers", Duration::from_secs(10), || { let client = client.clone(); async move { client .list_daemons() .await .map(|ds| ds.iter().any(|d| d.daemon_id == "restarter")) .unwrap_or(false) } }) .await; wait_for_async( "session reappears in catalog", Duration::from_secs(10), || { let client = client.clone(); let id = s.id; async move { client .list_sessions() .await .map(|ss| ss.iter().any(|x| x.id == id)) .unwrap_or(false) } }, ) .await; client.kill_session(s.id).await.expect("kill"); } #[tokio::test(flavor = "multi_thread")] async fn concurrent_spawns_each_get_unique_ids() { let (_reg, _daemons, client) = bring_up(&["parallel"]).await; let cwd = TempDir::new().unwrap(); let mut handles = Vec::new(); for _ in 0..5 { let client = client.clone(); let path = cwd.path().to_path_buf(); handles.push(tokio::spawn(async move { client .spawn_session(&create_req("parallel", &path, "sleep 30")) .await .expect("spawn") })); } let mut ids = Vec::new(); for h in handles { ids.push(h.await.unwrap().id); } ids.sort(); ids.dedup(); assert_eq!(ids.len(), 5, "expected 5 distinct session ids"); for id in ids { client.kill_session(id).await.expect("kill"); } } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- fn init_repo(dir: &std::path::Path) { run_git(&["init", "-q", "-b", "main"], dir); run_git(&["config", "user.email", "e2e@example.com"], dir); run_git(&["config", "user.name", "E2E"], dir); std::fs::write(dir.join("README"), b"hello\n").unwrap(); run_git(&["add", "."], dir); run_git(&["commit", "-q", "-m", "initial"], dir); } fn run_git(args: &[&str], cwd: &std::path::Path) { let out = Command::new("git") .arg("-C") .arg(cwd) .args(args) .output() .expect("git spawn"); assert!( out.status.success(), "git {args:?} failed: {}", String::from_utf8_lossy(&out.stderr) ); } fn git_worktree_list(dir: &std::path::Path) -> Vec { let out = Command::new("git") .arg("-C") .arg(dir) .args(["worktree", "list", "--porcelain"]) .output() .expect("git worktree list"); String::from_utf8_lossy(&out.stdout) .lines() .map(str::to_string) .collect() } #[tokio::test(flavor = "multi_thread")] async fn broker_t2_attach_through_home_daemon() { // NAP-247: two daemons share perimeter "wg-test". alpha's broker resolves // a session owned by beta, gets a pair JWT + perimeter address from the // registry, dials beta via T2, and proxies the attach. let registry = Registry::spawn("e2e-shared-token").await; let beta_bind = std::net::SocketAddr::from(( std::net::Ipv4Addr::LOCALHOST, maestro_e2e::util::free_port(), )); let beta = Daemon::spawn_at( "beta", ®istry.http_base(), "e2e-shared-token", beta_bind, DaemonOptions { perimeter_id: Some("wg-test".into()), private_addr: Some(beta_bind.to_string()), allow_inbound: vec!["alpha".into()], }, ) .await; let alpha_bind = std::net::SocketAddr::from(( std::net::Ipv4Addr::LOCALHOST, maestro_e2e::util::free_port(), )); let alpha = Daemon::spawn_at( "alpha", ®istry.http_base(), "e2e-shared-token", alpha_bind, DaemonOptions { perimeter_id: Some("wg-test".into()), private_addr: Some(alpha_bind.to_string()), allow_inbound: vec![], }, ) .await; assert_eq!(beta.bind_addr, beta_bind); assert_eq!(alpha.bind_addr, alpha_bind); let client = TestClient::new(®istry.http_base(), ®istry.ws_base()); client.bootstrap().await.expect("bootstrap"); wait_for_async("both daemons connected", Duration::from_secs(10), || { let client = client.clone(); async move { client .list_daemons() .await .map(|ds| { ds.iter().any(|d| d.daemon_id == "alpha") && ds.iter().any(|d| d.daemon_id == "beta") }) .unwrap_or(false) } }) .await; let cwd = TempDir::new().unwrap(); let session = client .spawn_session(&create_req("beta", cwd.path(), "bash --noprofile --norc")) .await .expect("spawn beta"); // Wait until alpha's catalog mirror sees the peer session — otherwise the // broker can't locate the owning daemon. let alpha_base = format!("http://{}", alpha.bind_addr); let sid = session.id; wait_for_async( "alpha mirrors beta's session", Duration::from_secs(5), || { let base = alpha_base.clone(); async move { let resp = match reqwest::get(format!("{base}/api/sessions")).await { Ok(r) => r, Err(_) => return false, }; let list: Vec = match resp.json().await { Ok(v) => v, Err(_) => return false, }; list.iter().any(|s| s.id == sid && s.daemon_id == "beta") } }, ) .await; // Attach through alpha's broker. let broker_url = format!("ws://{}/api/attach/{}", alpha.bind_addr, sid); let (mut ws, _) = tokio_tungstenite::connect_async(broker_url) .await .expect("broker WS handshake"); ws.send(WsMessage::Binary(b"echo MAESTRO_BROKER_E2E\n".to_vec())) .await .expect("send keys"); let want = b"MAESTRO_BROKER_E2E"; let mut buf = Vec::new(); let deadline = std::time::Instant::now() + Duration::from_secs(5); while std::time::Instant::now() < deadline { tokio::select! { msg = ws.next() => match msg { Some(Ok(WsMessage::Binary(b))) => buf.extend(b), Some(Ok(_)) => {} _ => break, }, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } if buf.windows(want.len()).any(|w| w == want) { break; } } assert!( buf.windows(want.len()).any(|w| w == want), "broker WS never echoed our keystrokes: buf len {}", buf.len() ); let _ = ws.close(None).await; client.kill_session(sid).await.expect("kill"); drop(alpha); drop(beta); drop(registry); } #[tokio::test(flavor = "multi_thread")] async fn broker_rejects_when_target_doesnt_allow_inbound() { // NAP-247: beta has an empty allow_inbound, so beta's // /sessions/:id/stream rejects alpha's broker JWT even though the // registry happily mints one (default deny is target-side only). let registry = Registry::spawn("e2e-shared-token").await; let beta_bind = std::net::SocketAddr::from(( std::net::Ipv4Addr::LOCALHOST, maestro_e2e::util::free_port(), )); let beta = Daemon::spawn_at( "beta", ®istry.http_base(), "e2e-shared-token", beta_bind, DaemonOptions { perimeter_id: Some("wg-test".into()), private_addr: Some(beta_bind.to_string()), // Default deny — alpha not listed. allow_inbound: vec![], }, ) .await; let alpha_bind = std::net::SocketAddr::from(( std::net::Ipv4Addr::LOCALHOST, maestro_e2e::util::free_port(), )); let alpha = Daemon::spawn_at( "alpha", ®istry.http_base(), "e2e-shared-token", alpha_bind, DaemonOptions { perimeter_id: Some("wg-test".into()), private_addr: Some(alpha_bind.to_string()), allow_inbound: vec![], }, ) .await; assert_eq!(beta.bind_addr, beta_bind); assert_eq!(alpha.bind_addr, alpha_bind); let client = TestClient::new(®istry.http_base(), ®istry.ws_base()); client.bootstrap().await.expect("bootstrap"); wait_for_async("both daemons connected", Duration::from_secs(10), || { let client = client.clone(); async move { client .list_daemons() .await .map(|ds| ds.len() >= 2) .unwrap_or(false) } }) .await; let cwd = TempDir::new().unwrap(); let session = client .spawn_session(&create_req("beta", cwd.path(), "sleep 30")) .await .expect("spawn beta"); let alpha_base = format!("http://{}", alpha.bind_addr); let sid = session.id; wait_for_async( "alpha mirrors beta's session", Duration::from_secs(5), || { let base = alpha_base.clone(); async move { let resp = match reqwest::get(format!("{base}/api/sessions")).await { Ok(r) => r, Err(_) => return false, }; let list: Vec = match resp.json().await { Ok(v) => v, Err(_) => return false, }; list.iter().any(|s| s.id == sid) } }, ) .await; let broker_url = format!("ws://{}/api/attach/{}", alpha.bind_addr, sid); // The handshake itself succeeds (alpha's broker accepts and starts // proxying), but beta will reject the pair JWT — the upstream WS closes // immediately, and alpha's proxy closes the client WS in turn. let (mut ws, _) = tokio_tungstenite::connect_async(broker_url) .await .expect("alpha handshake"); // We should get a close shortly without any binary frames. let deadline = std::time::Instant::now() + Duration::from_secs(3); let mut got_binary = false; while std::time::Instant::now() < deadline { tokio::select! { msg = ws.next() => match msg { Some(Ok(WsMessage::Binary(_))) => got_binary = true, None | Some(Err(_)) => break, Some(Ok(_)) => {} }, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } } assert!( !got_binary, "beta rejected the JWT but alpha somehow forwarded PTY bytes" ); client.kill_session(sid).await.expect("kill"); drop(alpha); drop(beta); drop(registry); } #[tokio::test(flavor = "multi_thread")] async fn daemon_events_ws_streams_snapshot_then_session_added() { // NAP-248: subscribe to a daemon's WS /api/events, expect an initial // Snapshot, then a SessionAdded after we spawn a session via REST. let (_reg, daemons, client) = bring_up(&["alpha"]).await; let cwd = TempDir::new().unwrap(); let alpha_addr = daemons[0].bind_addr; let ws_url = format!("ws://{alpha_addr}/api/events"); let (mut ws, _) = tokio_tungstenite::connect_async(ws_url) .await .expect("events WS handshake"); // First frame must be a Snapshot (the WS sends it eagerly on connect). let snapshot = next_event(&mut ws).await; match &snapshot { maestro_protocol::RegistryEvent::Snapshot { daemons, sessions } => { assert!(daemons.iter().any(|d| d.daemon_id == "alpha")); assert!(sessions.is_empty()); } other => panic!("expected initial Snapshot, got {other:?}"), } let session = client .spawn_session(&create_req("alpha", cwd.path(), "sleep 30")) .await .expect("spawn alpha"); // SessionAdded must show up on the WS within a reasonable window. let saw_added = tokio::time::timeout(Duration::from_secs(5), async { loop { match next_event(&mut ws).await { maestro_protocol::RegistryEvent::SessionAdded { session: s } if s.id == session.id => { return true; } _ => continue, } } }) .await .unwrap_or(false); assert!( saw_added, "events WS never delivered SessionAdded for own session" ); client.kill_session(session.id).await.expect("kill"); } async fn next_event(ws: &mut S) -> maestro_protocol::RegistryEvent where S: StreamExt> + Unpin, { loop { match ws.next().await { Some(Ok(WsMessage::Text(t))) => { return serde_json::from_str(&t).expect("RegistryEvent") } Some(Ok(_)) => continue, other => panic!("WS closed waiting for event: {other:?}"), } } }