Add reconcile loop scheduler #69

Merged
naps62-yolo merged 5 commits from issue/21-reconcile-loop into main 2026-08-22 22:00:12 +01:00
2 changed files with 73 additions and 22 deletions
Showing only changes of commit d8455df62d - Show all commits
+21 -10
View File
@@ -118,18 +118,29 @@ async fn run() -> Result<(), Error> {
tracing::info!(addr = %config.bind_addr, docs = arr_api::DOCS_PATH, "listening");
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
let reconcile_task = tokio::spawn(reconcile.run(shutdown_rx));
let mut reconcile_task = tokio::spawn(reconcile.run(shutdown_rx));
let signal_tx = shutdown_tx.clone();
let result = axum::serve(listener, app)
.with_graceful_shutdown(async move {
shutdown().await;
let _ = signal_tx.send(true);
})
.await;
let server = async move {
axum::serve(listener, app)
.with_graceful_shutdown(async move {
shutdown().await;
let _ = signal_tx.send(true);
})
.await
};
tokio::pin!(server);
naps62-yolo marked this conversation as resolved Outdated
Outdated
Review

A panic inside any action aborts the whole spawned task, taking all four lanes with it. Nothing observes that: axum::serve keeps running, no log is emitted, and the JoinError only surfaces here — after shutdown. The daemon then serves HTTP for hours with reconciliation dead and looks healthy.

Either catch per-action panics in run_tick (AssertUnwindSafe(...).catch_unwind(), log, count as a failure), or select over reconcile_task alongside serve so the process exits when the loop dies.

A panic inside any action aborts the whole spawned task, taking all four lanes with it. Nothing observes that: `axum::serve` keeps running, no log is emitted, and the `JoinError` only surfaces here — after shutdown. The daemon then serves HTTP for hours with reconciliation dead and looks healthy. Either catch per-action panics in `run_tick` (`AssertUnwindSafe(...).catch_unwind()`, log, count as a failure), or select over `reconcile_task` alongside serve so the process exits when the loop dies. <!-- agent-meta: {"model":"claude-opus-5","session":"283b4fd2"} -->
let _ = shutdown_tx.send(true);
reconcile_task.await?;
result.map_err(Error::Serve)
tokio::select! {
result = &mut server => {
let _ = shutdown_tx.send(true);
reconcile_task.await?;
result.map_err(Error::Serve)
}
result = &mut reconcile_task => {
result?;
server.await.map_err(Error::Serve)
}
}
}
/// Stop accepting on Ctrl-C, or on the SIGTERM a service manager sends.
+52 -12
View File
@@ -20,6 +20,7 @@ const RECONCILE_START_DELAY: Duration = Duration::ZERO;
const METADATA_START_DELAY: Duration = Duration::from_secs(5);
const REAPER_START_DELAY: Duration = Duration::from_secs(10);
const RSS_START_DELAY: Duration = Duration::from_secs(20);
const ACTION_TIMEOUT: Duration = Duration::from_secs(25);
naps62-yolo marked this conversation as resolved Outdated
Outdated
Review

One 25 s timeout covers every lane. It fits reconcile (30 s interval), but the metadata lane runs daily and a full TMDB refresh over the library is minutes of work, so that lane will time out on every tick and never complete once there is real data behind it — logged as a failure with no partial progress.

Make the timeout per-lane, e.g. derived from the lane interval (interval - slack, or an explicit value per Tick), rather than one constant.

One 25 s timeout covers every lane. It fits reconcile (30 s interval), but the metadata lane runs daily and a full TMDB refresh over the library is minutes of work, so that lane will time out on every tick and never complete once there is real data behind it — logged as a failure with no partial progress. Make the timeout per-lane, e.g. derived from the lane interval (`interval - slack`, or an explicit value per `Tick`), rather than one constant. <!-- agent-meta: {"model":"claude-opus-5","session":"283b4fd2"} -->
pub type ActionError = Box<dyn Error + Send + Sync>;
pub type ActionFuture<'a> =
@@ -124,6 +125,7 @@ impl Schedule {
pub struct ReconcileLoop {
database: Db,
schedule: Schedule,
action_timeout: Duration,
actions: Vec<RegisteredAction>,
}
@@ -133,6 +135,7 @@ impl ReconcileLoop {
Self {
database,
schedule: Schedule::default(),
action_timeout: ACTION_TIMEOUT,
actions: Vec::new(),
}
}
1
@@ -194,8 +197,10 @@ impl ReconcileLoop {
let mut report = TickReport::default();
for registered in self.actions.iter().filter(|action| action.tick == tick) {
match registered.action.run(&self.database).await {
Ok(outcomes) => {
match tokio::time::timeout(self.action_timeout, registered.action.run(&self.database))
.await
{
Ok(Ok(outcomes)) => {
for outcome in outcomes {
tracing::info!(
tick = %tick,
1
@@ -205,10 +210,9 @@ impl ReconcileLoop {
"reconcile action"
);
report.actions_taken += 1;
report.gaps_found += 1;
}
}
Err(error) => {
Ok(Err(error)) => {
report.failures += 1;
tracing::error!(
tick = %tick,
@@ -217,23 +221,34 @@ impl ReconcileLoop {
"reconcile action failed"
);
}
Err(_) => {
report.failures += 1;
tracing::error!(
tick = %tick,
worker = registered.action.name(),
naps62-yolo marked this conversation as resolved Outdated
Outdated
Review

This fires on every tick regardless of outcome, so an idle instance writes ~2880 all-zero lines a day from the reconcile lane alone, plus the other three. Log at info only when gaps_found + failures > 0, and drop the quiet case to debug.

This fires on every tick regardless of outcome, so an idle instance writes ~2880 all-zero lines a day from the reconcile lane alone, plus the other three. Log at info only when `gaps_found + failures > 0`, and drop the quiet case to debug. <!-- agent-meta: {"model":"claude-opus-5","session":"283b4fd2"} -->
timeout_seconds = self.action_timeout.as_secs(),
"reconcile action timed out"
);
}
}
}
tracing::info!(
tick = %tick,
gaps_found = report.gaps_found,
actions_taken = report.actions_taken,
failures = report.failures,
"reconcile tick"
);
if report.actions_taken == 0 && report.failures == 0 {
tracing::debug!(tick = %tick, "reconcile tick idle");
} else {
tracing::info!(
tick = %tick,
actions_taken = report.actions_taken,
failures = report.failures,
"reconcile tick"
);
}
report
}
}
#[derive(Debug, Default, PartialEq, Eq)]
struct TickReport {
gaps_found: usize,
actions_taken: usize,
failures: usize,
}
@@ -277,6 +292,19 @@ mod tests {
fail_after_add: bool,
}
#[derive(Debug)]
struct StalledAction;
impl Action for StalledAction {
fn name(&self) -> &'static str {
"stalled"
}
fn run<'a>(&'a self, _database: &'a Db) -> ActionFuture<'a> {
Box::pin(std::future::pending())
}
}
impl Action for FakeGrabAction {
fn name(&self) -> &'static str {
"fake-grab"
@@ -413,4 +441,16 @@ mod tests {
.unwrap();
assert_eq!(grabs, 1);
}
#[tokio::test]
async fn a_stalled_action_times_out() {
let (_directory, database, _release_id) = seeded_database().await;
let mut reconcile = ReconcileLoop::new(database).register(Tick::Reconcile, StalledAction);
reconcile.action_timeout = Duration::from_millis(1);
let report = reconcile.run_tick(Tick::Reconcile).await;
assert_eq!(report.failures, 1);
assert_eq!(report.actions_taken, 0);
}
}