Merge remote-tracking branch 'origin/main' into issue/22-grab-pipeline
ci / rust (pull_request) Successful in 1m34s
ci / web (pull_request) Successful in 37s
e2e / e2e (pull_request) Successful in 2m5s

# Conflicts:
#	.sqlx/query-6e988f7e8d8ac2167c589a35626b11ab88e1fdcc28c432da4c3debf1202eee7a.json
#	crates/arr-api/src/search.rs
This commit is contained in:
Miguel Palhas
2026-08-22 22:27:13 +01:00
16 changed files with 1683 additions and 104 deletions
+137 -14
View File
@@ -15,10 +15,10 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use arr_core::policy::{evaluate, Candidate};
use arr_core::{score::score, Language, MovieOverrides, Policy, Rule, Verdict};
use arr_core::{score::score, Language, Policy, Rule, TitleOverrides, Verdict};
use arr_db::{Db, MoviePolicy};
use arr_dl::{AddTorrent, TorrentSource, TransmissionClient};
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
@@ -31,6 +31,16 @@ use crate::reconcile::{Action, ActionFuture, Outcome};
/// and release-date gating are issue #26.
const MOVIES_PER_TICK: i64 = 5;
/// How long a discovered indexer list is reused. Prowlarr enumerates one
/// indexer per call plus a `t=caps` probe each, all sequential, so paying for
/// it every 30 s would leave the tick no room to search.
const INDEXER_CACHE_TTL: Duration = Duration::from_mins(15);
/// The share of the reconcile lane's 25 s budget discovery may spend. One
/// unresponsive tracker must not cancel the whole action before a single
/// targeted search has run; past this the last known list is used instead.
const INDEXER_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(8);
/// Seeding obligations, per tracker in principle (§7.3) and per install in
/// practice until issue #25 gives them a home. Both are set on the torrent at
/// add time and enforced by Transmission.
@@ -56,6 +66,15 @@ pub enum GrabError {
name: String,
source: serde_json::Error,
},
#[error("indexer discovery timed out after {0:?} and none is known yet")]
IndexerDiscoveryTimeout(Duration),
}
/// The searchable indexer ids, and when they were last discovered.
#[derive(Debug, Default)]
struct IndexerCache {
ids: Vec<i64>,
refreshed_at: Option<Instant>,
}
/// Sends the best eligible release for every wanted movie that has neither a
@@ -66,6 +85,10 @@ pub struct GrabAction {
transmission: TransmissionClient,
download_dir: PathBuf,
seeding: SeedingLimits,
indexers: tokio::sync::RwLock<IndexerCache>,
/// [`INDEXER_DISCOVERY_TIMEOUT`], overridden by tests that cannot wait
/// out the real one. Mirrors `ReconcileLoop`'s action timeout override.
discovery_timeout: Duration,
}
impl GrabAction {
@@ -81,6 +104,8 @@ impl GrabAction {
transmission,
download_dir,
seeding,
indexers: tokio::sync::RwLock::new(IndexerCache::default()),
discovery_timeout: INDEXER_DISCOVERY_TIMEOUT,
}
}
@@ -91,12 +116,7 @@ impl GrabAction {
return Ok(outcomes);
}
let indexers = self.prowlarr.indexers().await?;
let searchable: Vec<i64> = indexers
.iter()
.filter(|indexer| indexer.capabilities.search.available)
.map(|indexer| indexer.id)
.collect();
let searchable = self.searchable_indexers().await?;
if searchable.is_empty() {
tracing::warn!("no indexer advertises a text search; nothing can be grabbed");
return Ok(outcomes);
@@ -173,6 +193,51 @@ impl GrabAction {
Ok(outcomes)
}
/// The indexers that accept a text search, cached across ticks.
///
/// Discovery is bounded and its result is reused, because Prowlarr probes
/// capabilities one indexer at a time: a single slow tracker would
/// otherwise burn the whole reconcile budget before any search runs. A
/// refresh that fails or times out keeps the previous list rather than
/// stopping the tick, and only an empty cache turns that into an error.
async fn searchable_indexers(&self) -> Result<Vec<i64>, GrabError> {
{
let cache = self.indexers.read().await;
if cache
.refreshed_at
.is_some_and(|at| at.elapsed() < INDEXER_CACHE_TTL)
{
return Ok(cache.ids.clone());
}
}
let discovered =
tokio::time::timeout(self.discovery_timeout, self.prowlarr.indexers()).await;
let mut cache = self.indexers.write().await;
match discovered {
Ok(Ok(indexers)) => {
cache.ids = indexers
.iter()
.filter(|indexer| indexer.capabilities.search.available)
.map(|indexer| indexer.id)
.collect();
cache.refreshed_at = Some(Instant::now());
}
Ok(Err(error)) if cache.ids.is_empty() => return Err(error.into()),
Err(_) if cache.ids.is_empty() => {
return Err(GrabError::IndexerDiscoveryTimeout(self.discovery_timeout))
}
Ok(Err(error)) => {
tracing::warn!(%error, "indexer discovery failed; using the last known list");
}
Err(_) => tracing::warn!(
timeout_seconds = self.discovery_timeout.as_secs(),
"indexer discovery timed out; using the last known list"
),
}
Ok(cache.ids.clone())
}
/// Search every indexer for one title, cache each candidate with its
/// verdict and score (§9.3), and return the eligible ones best first.
///
@@ -404,7 +469,7 @@ async fn store_release(
movie_id: i64,
release: &SearchRelease,
policy: &Policy,
overrides: &MovieOverrides,
overrides: &TitleOverrides,
original_language: &Language,
) -> Result<Option<Eligible>, GrabError> {
let parsed = arr_parse::parse(&release.name);
@@ -415,13 +480,20 @@ async fn store_release(
Candidate::PreGrab(&parsed),
release.size,
);
let score = score(
let scored = score(
policy,
Candidate::PreGrab(&parsed),
release.size.unwrap_or(0),
release.seeders.unwrap_or(0),
)
.total;
release.size.unwrap_or_default(),
release.seeders.unwrap_or_default(),
);
// A release that did not say its size is not a tiny one: scoring it
// against the band's floor would bury it. Same treatment as the manual
// search view, so the ranking the operator sees is the one that picks.
let score = if release.size.is_some() {
scored.total
} else {
scored.source.saturating_add(scored.seeders)
};
let (verdict, rule) = verdict_columns(&evaluation.verdict);
let parsed_json = serde_json::to_string(&parsed).map_err(|source| GrabError::Parsed {
name: release.name.clone(),
@@ -936,6 +1008,57 @@ mod tests {
assert!(grabs(&database).await.is_empty());
}
/// Prowlarr probes capabilities one indexer at a time, so paying for
/// discovery every 30 s would leave the tick no room to search.
#[tokio::test]
async fn indexer_discovery_is_cached_across_ticks() {
let (_dir, database) = wanted_movie().await;
let indexer = prowlarr().await;
let (downloader, _fake) = transmission().await;
let action = action(&indexer, &downloader);
action.tick(&database).await.unwrap();
action.tick(&database).await.unwrap();
let enumerations = indexer
.received_requests()
.await
.unwrap()
.into_iter()
.filter(|request| request.url.path() == "/api/v1/indexer")
.count();
assert_eq!(enumerations, 1);
}
/// One unresponsive tracker must not cancel the action before a single
/// search has run.
#[tokio::test]
async fn a_stalled_discovery_gives_up_instead_of_hanging() {
let (_dir, database) = wanted_movie().await;
let indexer = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/indexer"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(Duration::from_secs(30))
.set_body_json(json!([])),
)
.mount(&indexer)
.await;
let (downloader, fake) = transmission().await;
let mut action = action(&indexer, &downloader);
action.discovery_timeout = Duration::from_millis(50);
let error = action
.tick(&database)
.await
.expect_err("discovery cannot complete");
assert!(matches!(error, GrabError::IndexerDiscoveryTimeout(_)));
assert!(fake.torrents().is_empty());
}
#[test]
fn indexer_links_and_magnets_both_reach_transmission() {
assert_eq!(