feat(maestro): add maestro-e2e cross-process test harness (NAP-217)
ci / rust (push) Successful in 2m8s

New `crates/maestro-e2e` exercises the full headless stack — registry,
two daemons, real HTTP+WS — as actual subprocesses.

Harness pieces:

- `Registry::spawn` and `Daemon::spawn` launch the real built binaries
  with `env_clear()` plus an isolated `$HOME`, `XDG_CONFIG_HOME`,
  `XDG_STATE_HOME`, and (for daemons) `TMUX_TMPDIR`. Drop guards kill
  the child and clean the temp dirs.
- `Daemon::stop` + `Daemon::restart` swap the process without losing
  the state dirs, so the recovery test exercises a real tmux
  re-adoption.
- A `build::workspace_target_bin` helper re-invokes cargo from the
  test process to build `maestrod` / `maestro-registry` on demand
  (cargo only ships libs as path-dep of e2e by default).
- `TestClient` wraps reqwest + tokio-tungstenite for the public
  HTTP+WS surface — list/spawn/kill/connect-info/attach + events WS.

Scenarios covered (`tests/e2e.rs`):

- list sessions from two daemons through one registry
- multi-attach: two WS clients see the same PTY output
- worktree with `cleanup_on_kill: true` is removed after kill
- daemon restart: catalog drops then repopulates from `recover()`
- concurrent spawns all get distinct ids

All five pass in ~0.7s in parallel; cleanup confirmed (no leftover
processes or temp dirs).
This commit is contained in:
2026-05-26 07:38:19 +00:00
parent a2ffb0ad33
commit 91f252b4e2
9 changed files with 990 additions and 0 deletions
+72
View File
@@ -0,0 +1,72 @@
//! Locate / build the maestro binaries needed by the harness.
//!
//! `cargo test -p maestro-e2e` does not, by default, rebuild bins from other
//! workspace crates. So we re-invoke cargo from the first test that needs a
//! binary and cache the resolved paths. The re-invocation acquires its own
//! workspace lock — by the time tests start, the outer cargo has already
//! released it.
use std::path::{Path, PathBuf};
use std::sync::OnceLock;
/// Absolute path of the workspace root (the dir containing the top-level
/// `Cargo.toml` whose body has `[workspace]`).
pub fn workspace_root() -> &'static Path {
static ROOT: OnceLock<PathBuf> = OnceLock::new();
ROOT.get_or_init(|| {
let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let mut cur: &Path = manifest.as_path();
loop {
let candidate = cur.join("Cargo.toml");
if candidate.exists() {
if let Ok(body) = std::fs::read_to_string(&candidate) {
if body.contains("[workspace]") {
return cur.to_path_buf();
}
}
}
match cur.parent() {
Some(p) => cur = p,
None => panic!(
"could not find workspace root starting from {}",
PathBuf::from(env!("CARGO_MANIFEST_DIR")).display()
),
}
}
})
}
/// Resolve `target/<profile>/<name>` under the workspace root, after running
/// `cargo build --bin <name> -p <package>` to make sure it exists.
pub fn workspace_target_bin(package: &str, bin: &str) -> PathBuf {
static BUILDS: OnceLock<std::sync::Mutex<Vec<String>>> = OnceLock::new();
let builds = BUILDS.get_or_init(|| std::sync::Mutex::new(Vec::new()));
let already = {
let lock = builds.lock().unwrap();
lock.iter().any(|b| b == bin)
};
let release = !cfg!(debug_assertions);
if !already {
let mut cmd = std::process::Command::new(env!("CARGO"));
cmd.args(["build", "--quiet", "-p", package, "--bin", bin]);
if release {
cmd.arg("--release");
}
let status = cmd
.current_dir(workspace_root())
.status()
.unwrap_or_else(|e| panic!("failed to invoke cargo build for {bin}: {e}"));
assert!(status.success(), "cargo build for `{bin}` failed");
builds.lock().unwrap().push(bin.to_string());
}
let profile = if release { "release" } else { "debug" };
let path = workspace_root().join("target").join(profile).join(bin);
assert!(
path.exists(),
"expected built binary at {} after `cargo build`",
path.display()
);
path
}
+231
View File
@@ -0,0 +1,231 @@
//! Thin HTTP+WS client used by tests to drive the registry.
//!
//! Cookies are managed automatically by `reqwest`'s built-in cookie store.
#![allow(clippy::result_large_err)]
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use maestro_protocol::{ConnectInfo, CreateSessionRequest, DaemonInfo, RegistryEvent, Session};
use reqwest::Client as HttpClient;
use std::sync::Arc;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message as WsMessage;
use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
use uuid::Uuid;
#[derive(Debug, thiserror::Error)]
#[allow(clippy::result_large_err)]
pub enum ClientError {
#[error("http: {0}")]
Http(#[from] reqwest::Error),
#[error("ws: {0}")]
Ws(#[from] tokio_tungstenite::tungstenite::Error),
#[error("json: {0}")]
Json(#[from] serde_json::Error),
#[error("unexpected status {status}: {body}")]
Status { status: u16, body: String },
}
/// Sticky-cookie HTTP client + a few WS-aware helpers.
#[derive(Debug, Clone)]
pub struct TestClient {
http_base: String,
ws_base: String,
http: HttpClient,
}
impl TestClient {
pub fn new(http_base: &str, ws_base: &str) -> Self {
let http = HttpClient::builder()
.cookie_store(true)
.timeout(Duration::from_secs(5))
.build()
.expect("reqwest client");
Self {
http_base: http_base.trim_end_matches('/').to_string(),
ws_base: ws_base.trim_end_matches('/').to_string(),
http,
}
}
/// Hit any GET endpoint so the registry issues the auth cookie that all
/// subsequent calls reuse.
pub async fn bootstrap(&self) -> Result<(), ClientError> {
let _ = self.list_daemons().await?;
Ok(())
}
pub async fn list_daemons(&self) -> Result<Vec<DaemonInfo>, ClientError> {
let r = self
.http
.get(format!("{}/api/daemons", self.http_base))
.send()
.await?;
check_ok(&r)?;
Ok(r.json().await?)
}
pub async fn list_sessions(&self) -> Result<Vec<Session>, ClientError> {
let r = self
.http
.get(format!("{}/api/sessions", self.http_base))
.send()
.await?;
check_ok(&r)?;
Ok(r.json().await?)
}
pub async fn spawn_session(&self, req: &CreateSessionRequest) -> Result<Session, ClientError> {
let r = self
.http
.post(format!("{}/api/sessions", self.http_base))
.json(req)
.send()
.await?;
let status = r.status();
if !status.is_success() {
let body = r.text().await.unwrap_or_default();
return Err(ClientError::Status {
status: status.as_u16(),
body,
});
}
Ok(r.json().await?)
}
pub async fn kill_session(&self, id: Uuid) -> Result<(), ClientError> {
let r = self
.http
.delete(format!("{}/api/sessions/{id}", self.http_base))
.send()
.await?;
check_ok(&r)?;
Ok(())
}
pub async fn connect_info(&self, id: Uuid) -> Result<ConnectInfo, ClientError> {
let r = self
.http
.get(format!("{}/api/sessions/{id}/connect-info", self.http_base))
.send()
.await?;
check_ok(&r)?;
Ok(r.json().await?)
}
/// Connect to the daemon's per-session WS and return the stream. The
/// caller drives the conversation.
pub async fn attach(
&self,
info: &ConnectInfo,
id: Uuid,
) -> Result<WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>, ClientError> {
let url = format!(
"{}/sessions/{id}/stream?token={}",
info.daemon_url.trim_end_matches('/'),
info.jwt
);
let request = url.into_client_request()?;
let (ws, _resp) = connect_async(request).await?;
Ok(ws)
}
/// Subscribe to the registry's `/api/events` WS. Returns a tokio task
/// handle that drives the WS and pushes parsed events into an Arc<Mutex<Vec>>.
pub async fn subscribe_events(&self) -> EventStream {
// Forward the latest cookie value with the upgrade — `reqwest` and
// `tungstenite` don't share cookie jars, so we copy the cookie out by
// hand. The registry will accept us either way (missing cookie just
// gets a fresh one), but reusing the same identity keeps log noise
// down.
let cookie_header = self.grab_cookie().await.unwrap_or_default();
let url = format!("{}/api/events", self.ws_base);
let mut request = url.into_client_request().expect("ws req");
if !cookie_header.is_empty() {
request.headers_mut().insert(
reqwest::header::COOKIE,
cookie_header.parse().expect("cookie hv"),
);
}
let (ws, _resp) = connect_async(request).await.expect("events ws");
let events = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let stream_events = events.clone();
let (mut sink, mut stream) = ws.split();
let task = tokio::spawn(async move {
while let Some(msg) = stream.next().await {
match msg {
Ok(WsMessage::Text(t)) => {
if let Ok(ev) = serde_json::from_str::<RegistryEvent>(&t) {
stream_events.lock().await.push(ev);
}
}
Ok(WsMessage::Close(_)) | Err(_) => break,
_ => {}
}
}
let _ = sink.close().await;
});
EventStream { events, task }
}
async fn grab_cookie(&self) -> Option<String> {
// Force a request to make sure the jar has a cookie, then read it back
// via a debug header roundtrip. reqwest's CookieStore isn't directly
// exposed; the simplest path is to ask the registry for an auth
// cookie and capture the Set-Cookie from the response.
let r = self
.http
.get(format!("{}/api/daemons", self.http_base))
.send()
.await
.ok()?;
// The cookie store has the cookie now, but reqwest doesn't surface
// it. We rebuild from Set-Cookie if present.
r.headers()
.get_all(reqwest::header::SET_COOKIE)
.iter()
.find_map(|hv| hv.to_str().ok().and_then(extract_cookie_pair))
}
}
fn extract_cookie_pair(set_cookie: &str) -> Option<String> {
// Take everything before the first ';', which is `name=value`.
set_cookie
.split(';')
.next()
.map(str::trim)
.map(String::from)
}
fn check_ok(r: &reqwest::Response) -> Result<(), ClientError> {
let status = r.status();
if !status.is_success() {
return Err(ClientError::Status {
status: status.as_u16(),
body: format!("HTTP {status}"),
});
}
Ok(())
}
/// Handle to a background task draining the events WS into a vector.
#[derive(Debug)]
pub struct EventStream {
events: Arc<tokio::sync::Mutex<Vec<RegistryEvent>>>,
task: tokio::task::JoinHandle<()>,
}
impl EventStream {
pub async fn snapshot(&self) -> Vec<RegistryEvent> {
self.events.lock().await.clone()
}
}
impl Drop for EventStream {
fn drop(&mut self) {
self.task.abort();
}
}
+165
View File
@@ -0,0 +1,165 @@
//! Subprocess wrapper around `maestrod`.
use std::net::{Ipv4Addr, SocketAddr};
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::Duration;
use tempfile::TempDir;
use crate::build::workspace_target_bin;
use crate::util::{free_port, wait_for_async};
/// A live `maestrod` running in a child process. Kill the child on drop.
///
/// `state_dir`, `home`, and `tmux_tmpdir` are kept as `TempDir`s so they
/// outlive the process and get cleaned up automatically.
#[derive(Debug)]
pub struct Daemon {
pub daemon_id: String,
pub bind_addr: SocketAddr,
pub daemon_url: String,
child: Option<Child>,
home: TempDir,
state: TempDir,
tmux: TempDir,
registry_url: String,
token: String,
}
impl Daemon {
/// Spawn a fresh daemon pointing at `registry_url` (the daemon-facing WS
/// URL — usually `ws://<addr>/daemon`).
pub async fn spawn(daemon_id: &str, registry_url: &str, token: &str) -> Self {
let port = free_port();
let bind = SocketAddr::from((Ipv4Addr::LOCALHOST, port));
Self::spawn_with_port(daemon_id, registry_url, token, bind).await
}
/// Stop the underlying process, but keep its state dirs around so we can
/// `restart()` against the same `$HOME` / `state` / `TMUX_TMPDIR`.
pub fn stop(&mut self) {
if let Some(mut c) = self.child.take() {
let _ = c.kill();
let _ = c.wait();
}
}
/// Re-spawn the daemon on the same port + state. The previous tmux
/// server (under `TMUX_TMPDIR`) is left running so `recover()` can adopt
/// its sessions.
pub async fn restart(&mut self) {
self.stop();
let child = launch(
&self.daemon_id,
self.bind_addr,
&self.daemon_url,
&self.registry_url,
&self.token,
self.home.path(),
self.state.path(),
self.tmux.path(),
);
self.child = Some(child);
self.wait_until_attach_ready().await;
}
async fn spawn_with_port(
daemon_id: &str,
registry_url: &str,
token: &str,
bind: SocketAddr,
) -> Self {
let home = TempDir::new().expect("home tmpdir");
let state = TempDir::new().expect("state tmpdir");
let tmux = TempDir::new().expect("tmux tmpdir");
let daemon_url = format!("ws://{bind}");
let child = launch(
daemon_id,
bind,
&daemon_url,
registry_url,
token,
home.path(),
state.path(),
tmux.path(),
);
let me = Self {
daemon_id: daemon_id.to_string(),
bind_addr: bind,
daemon_url,
child: Some(child),
home,
state,
tmux,
registry_url: registry_url.to_string(),
token: token.to_string(),
};
me.wait_until_attach_ready().await;
me
}
async fn wait_until_attach_ready(&self) {
let addr = self.bind_addr;
wait_for_async(
"daemon attach port",
Duration::from_secs(5),
|| async move { tokio::net::TcpStream::connect(addr).await.is_ok() },
)
.await;
}
pub fn home(&self) -> &std::path::Path {
self.home.path()
}
pub fn state_dir(&self) -> PathBuf {
// Matches maestro_daemon::default_state_dir which uses XDG_STATE_HOME.
self.home.path().join(".local/state/maestrod")
}
}
impl Drop for Daemon {
fn drop(&mut self) {
if let Some(mut c) = self.child.take() {
let _ = c.kill();
let _ = c.wait();
}
}
}
#[allow(clippy::too_many_arguments)]
fn launch(
daemon_id: &str,
bind: SocketAddr,
daemon_url: &str,
registry_url: &str,
token: &str,
home: &std::path::Path,
_state: &std::path::Path,
tmux_tmpdir: &std::path::Path,
) -> Child {
let bin = workspace_target_bin("maestro-daemon", "maestrod");
Command::new(&bin)
.env_clear()
.env(
"PATH",
std::env::var("PATH").unwrap_or_else(|_| "/usr/bin:/bin".into()),
)
.env("HOME", home)
.env("XDG_CONFIG_HOME", home.join(".config"))
.env("XDG_STATE_HOME", home.join(".local/state"))
.env("TMUX_TMPDIR", tmux_tmpdir)
.env("MAESTRO_TOKEN", token)
.env("MAESTROD_DAEMON_ID", daemon_id)
.env("MAESTROD_REGISTRY_URL", registry_url)
.env("MAESTROD_BIND_ADDR", bind.to_string())
.env("MAESTROD_DAEMON_URL", daemon_url)
.env("RUST_LOG", "warn")
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.arg("run")
.spawn()
.unwrap_or_else(|e| panic!("spawn {} failed: {e}", bin.display()))
}
+20
View File
@@ -0,0 +1,20 @@
//! Cross-process integration test harness for Maestro.
//!
//! Each test spawns a real `maestro-registry` and one or more `maestrod`
//! subprocesses, isolates them in temp dirs (`$HOME`, `TMUX_TMPDIR`, state
//! dir), and drives them through their public HTTP/WS APIs.
//!
//! See `tests/` for the actual scenarios.
#![deny(missing_debug_implementations)]
pub mod build;
pub mod client;
pub mod daemon;
pub mod registry;
pub mod util;
pub use build::workspace_target_bin;
pub use client::TestClient;
pub use daemon::Daemon;
pub use registry::Registry;
+94
View File
@@ -0,0 +1,94 @@
//! Subprocess wrapper around `maestro-registry`.
use std::net::{Ipv4Addr, SocketAddr};
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::Duration;
use tempfile::TempDir;
use crate::build::workspace_target_bin;
use crate::util::{free_port, wait_for_async};
/// A `maestro-registry` running in a child process. The state directory is
/// kept as a `TempDir` so it survives the `Registry`'s lifetime and gets
/// cleaned up on drop.
#[derive(Debug)]
pub struct Registry {
pub addr: SocketAddr,
pub token: String,
child: Option<Child>,
_home: TempDir,
}
impl Registry {
pub async fn spawn(token: &str) -> Self {
let port = free_port();
let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port));
let home = TempDir::new().expect("home tmpdir");
let bin = workspace_target_bin("maestro-registry", "maestro-registry");
let child = Command::new(&bin)
.env_clear()
// PATH so tmux etc. resolve. We need a few essentials.
.env(
"PATH",
std::env::var("PATH").unwrap_or_else(|_| "/usr/bin:/bin".into()),
)
.env("HOME", home.path())
.env("XDG_CONFIG_HOME", home.path().join(".config"))
.env("XDG_STATE_HOME", home.path().join(".local/state"))
.env("MAESTRO_TOKEN", token)
.env("MAESTRO_REGISTRY_BIND_ADDR", addr.to_string())
.env("RUST_LOG", "warn")
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.spawn()
.unwrap_or_else(|e| panic!("spawn {} failed: {e}", bin.display()));
let url = format!("http://{addr}/api/daemons");
let http = reqwest::Client::builder()
.timeout(Duration::from_secs(2))
.build()
.expect("reqwest client");
wait_for_async("registry to bind", Duration::from_secs(5), || {
let http = http.clone();
let url = url.clone();
async move { http.get(&url).send().await.is_ok() }
})
.await;
Self {
addr,
token: token.to_string(),
child: Some(child),
_home: home,
}
}
pub fn http_base(&self) -> String {
format!("http://{}", self.addr)
}
pub fn ws_base(&self) -> String {
format!("ws://{}", self.addr)
}
/// URL daemons dial.
pub fn daemon_ws_url(&self) -> String {
format!("ws://{}/daemon", self.addr)
}
pub fn state_dir(&self) -> PathBuf {
self._home.path().to_path_buf()
}
}
impl Drop for Registry {
fn drop(&mut self) {
if let Some(mut c) = self.child.take() {
let _ = c.kill();
let _ = c.wait();
}
}
}
+31
View File
@@ -0,0 +1,31 @@
//! Helpers shared between harness modules and tests.
use std::future::Future;
use std::net::{Ipv4Addr, SocketAddr, TcpListener};
use std::time::{Duration, Instant};
/// Pick a free TCP port by binding to `127.0.0.1:0` and immediately releasing.
///
/// There is a small race: another process may grab the port between drop and
/// the child re-binding, but for local single-user test runs it's negligible.
pub fn free_port() -> u16 {
let l = TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))).expect("bind :0");
l.local_addr().expect("local_addr").port()
}
/// Poll `cond` every 50ms until it returns `true` or `timeout` elapses.
/// Panics on timeout with `label`.
pub async fn wait_for_async<F, Fut>(label: &str, timeout: Duration, mut cond: F)
where
F: FnMut() -> Fut,
Fut: Future<Output = bool>,
{
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if cond().await {
return;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
panic!("timed out after {timeout:?} waiting for: {label}");
}