Merge remote-tracking branch 'origin/main' into issue/22-grab-pipeline
# Conflicts: # .sqlx/query-6e988f7e8d8ac2167c589a35626b11ab88e1fdcc28c432da4c3debf1202eee7a.json # crates/arr-api/src/search.rs
This commit is contained in:
+137
-14
@@ -15,10 +15,10 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arr_core::policy::{evaluate, Candidate};
|
||||
use arr_core::{score::score, Language, MovieOverrides, Policy, Rule, Verdict};
|
||||
use arr_core::{score::score, Language, Policy, Rule, TitleOverrides, Verdict};
|
||||
use arr_db::{Db, MoviePolicy};
|
||||
use arr_dl::{AddTorrent, TorrentSource, TransmissionClient};
|
||||
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
|
||||
@@ -31,6 +31,16 @@ use crate::reconcile::{Action, ActionFuture, Outcome};
|
||||
/// and release-date gating are issue #26.
|
||||
const MOVIES_PER_TICK: i64 = 5;
|
||||
|
||||
/// How long a discovered indexer list is reused. Prowlarr enumerates one
|
||||
/// indexer per call plus a `t=caps` probe each, all sequential, so paying for
|
||||
/// it every 30 s would leave the tick no room to search.
|
||||
const INDEXER_CACHE_TTL: Duration = Duration::from_mins(15);
|
||||
|
||||
/// The share of the reconcile lane's 25 s budget discovery may spend. One
|
||||
/// unresponsive tracker must not cancel the whole action before a single
|
||||
/// targeted search has run; past this the last known list is used instead.
|
||||
const INDEXER_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(8);
|
||||
|
||||
/// Seeding obligations, per tracker in principle (§7.3) and per install in
|
||||
/// practice until issue #25 gives them a home. Both are set on the torrent at
|
||||
/// add time and enforced by Transmission.
|
||||
@@ -56,6 +66,15 @@ pub enum GrabError {
|
||||
name: String,
|
||||
source: serde_json::Error,
|
||||
},
|
||||
#[error("indexer discovery timed out after {0:?} and none is known yet")]
|
||||
IndexerDiscoveryTimeout(Duration),
|
||||
}
|
||||
|
||||
/// The searchable indexer ids, and when they were last discovered.
|
||||
#[derive(Debug, Default)]
|
||||
struct IndexerCache {
|
||||
ids: Vec<i64>,
|
||||
refreshed_at: Option<Instant>,
|
||||
}
|
||||
|
||||
/// Sends the best eligible release for every wanted movie that has neither a
|
||||
@@ -66,6 +85,10 @@ pub struct GrabAction {
|
||||
transmission: TransmissionClient,
|
||||
download_dir: PathBuf,
|
||||
seeding: SeedingLimits,
|
||||
indexers: tokio::sync::RwLock<IndexerCache>,
|
||||
/// [`INDEXER_DISCOVERY_TIMEOUT`], overridden by tests that cannot wait
|
||||
/// out the real one. Mirrors `ReconcileLoop`'s action timeout override.
|
||||
discovery_timeout: Duration,
|
||||
}
|
||||
|
||||
impl GrabAction {
|
||||
@@ -81,6 +104,8 @@ impl GrabAction {
|
||||
transmission,
|
||||
download_dir,
|
||||
seeding,
|
||||
indexers: tokio::sync::RwLock::new(IndexerCache::default()),
|
||||
discovery_timeout: INDEXER_DISCOVERY_TIMEOUT,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,12 +116,7 @@ impl GrabAction {
|
||||
return Ok(outcomes);
|
||||
}
|
||||
|
||||
let indexers = self.prowlarr.indexers().await?;
|
||||
let searchable: Vec<i64> = indexers
|
||||
.iter()
|
||||
.filter(|indexer| indexer.capabilities.search.available)
|
||||
.map(|indexer| indexer.id)
|
||||
.collect();
|
||||
let searchable = self.searchable_indexers().await?;
|
||||
if searchable.is_empty() {
|
||||
tracing::warn!("no indexer advertises a text search; nothing can be grabbed");
|
||||
return Ok(outcomes);
|
||||
@@ -173,6 +193,51 @@ impl GrabAction {
|
||||
Ok(outcomes)
|
||||
}
|
||||
|
||||
/// The indexers that accept a text search, cached across ticks.
|
||||
///
|
||||
/// Discovery is bounded and its result is reused, because Prowlarr probes
|
||||
/// capabilities one indexer at a time: a single slow tracker would
|
||||
/// otherwise burn the whole reconcile budget before any search runs. A
|
||||
/// refresh that fails or times out keeps the previous list rather than
|
||||
/// stopping the tick, and only an empty cache turns that into an error.
|
||||
async fn searchable_indexers(&self) -> Result<Vec<i64>, GrabError> {
|
||||
{
|
||||
let cache = self.indexers.read().await;
|
||||
if cache
|
||||
.refreshed_at
|
||||
.is_some_and(|at| at.elapsed() < INDEXER_CACHE_TTL)
|
||||
{
|
||||
return Ok(cache.ids.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let discovered =
|
||||
tokio::time::timeout(self.discovery_timeout, self.prowlarr.indexers()).await;
|
||||
let mut cache = self.indexers.write().await;
|
||||
match discovered {
|
||||
Ok(Ok(indexers)) => {
|
||||
cache.ids = indexers
|
||||
.iter()
|
||||
.filter(|indexer| indexer.capabilities.search.available)
|
||||
.map(|indexer| indexer.id)
|
||||
.collect();
|
||||
cache.refreshed_at = Some(Instant::now());
|
||||
}
|
||||
Ok(Err(error)) if cache.ids.is_empty() => return Err(error.into()),
|
||||
Err(_) if cache.ids.is_empty() => {
|
||||
return Err(GrabError::IndexerDiscoveryTimeout(self.discovery_timeout))
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
tracing::warn!(%error, "indexer discovery failed; using the last known list");
|
||||
}
|
||||
Err(_) => tracing::warn!(
|
||||
timeout_seconds = self.discovery_timeout.as_secs(),
|
||||
"indexer discovery timed out; using the last known list"
|
||||
),
|
||||
}
|
||||
Ok(cache.ids.clone())
|
||||
}
|
||||
|
||||
/// Search every indexer for one title, cache each candidate with its
|
||||
/// verdict and score (§9.3), and return the eligible ones best first.
|
||||
///
|
||||
@@ -404,7 +469,7 @@ async fn store_release(
|
||||
movie_id: i64,
|
||||
release: &SearchRelease,
|
||||
policy: &Policy,
|
||||
overrides: &MovieOverrides,
|
||||
overrides: &TitleOverrides,
|
||||
original_language: &Language,
|
||||
) -> Result<Option<Eligible>, GrabError> {
|
||||
let parsed = arr_parse::parse(&release.name);
|
||||
@@ -415,13 +480,20 @@ async fn store_release(
|
||||
Candidate::PreGrab(&parsed),
|
||||
release.size,
|
||||
);
|
||||
let score = score(
|
||||
let scored = score(
|
||||
policy,
|
||||
Candidate::PreGrab(&parsed),
|
||||
release.size.unwrap_or(0),
|
||||
release.seeders.unwrap_or(0),
|
||||
)
|
||||
.total;
|
||||
release.size.unwrap_or_default(),
|
||||
release.seeders.unwrap_or_default(),
|
||||
);
|
||||
// A release that did not say its size is not a tiny one: scoring it
|
||||
// against the band's floor would bury it. Same treatment as the manual
|
||||
// search view, so the ranking the operator sees is the one that picks.
|
||||
let score = if release.size.is_some() {
|
||||
scored.total
|
||||
} else {
|
||||
scored.source.saturating_add(scored.seeders)
|
||||
};
|
||||
let (verdict, rule) = verdict_columns(&evaluation.verdict);
|
||||
let parsed_json = serde_json::to_string(&parsed).map_err(|source| GrabError::Parsed {
|
||||
name: release.name.clone(),
|
||||
@@ -936,6 +1008,57 @@ mod tests {
|
||||
assert!(grabs(&database).await.is_empty());
|
||||
}
|
||||
|
||||
/// Prowlarr probes capabilities one indexer at a time, so paying for
|
||||
/// discovery every 30 s would leave the tick no room to search.
|
||||
#[tokio::test]
|
||||
async fn indexer_discovery_is_cached_across_ticks() {
|
||||
let (_dir, database) = wanted_movie().await;
|
||||
let indexer = prowlarr().await;
|
||||
let (downloader, _fake) = transmission().await;
|
||||
let action = action(&indexer, &downloader);
|
||||
|
||||
action.tick(&database).await.unwrap();
|
||||
action.tick(&database).await.unwrap();
|
||||
|
||||
let enumerations = indexer
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|request| request.url.path() == "/api/v1/indexer")
|
||||
.count();
|
||||
assert_eq!(enumerations, 1);
|
||||
}
|
||||
|
||||
/// One unresponsive tracker must not cancel the action before a single
|
||||
/// search has run.
|
||||
#[tokio::test]
|
||||
async fn a_stalled_discovery_gives_up_instead_of_hanging() {
|
||||
let (_dir, database) = wanted_movie().await;
|
||||
let indexer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/v1/indexer"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(Duration::from_secs(30))
|
||||
.set_body_json(json!([])),
|
||||
)
|
||||
.mount(&indexer)
|
||||
.await;
|
||||
let (downloader, fake) = transmission().await;
|
||||
|
||||
let mut action = action(&indexer, &downloader);
|
||||
action.discovery_timeout = Duration::from_millis(50);
|
||||
|
||||
let error = action
|
||||
.tick(&database)
|
||||
.await
|
||||
.expect_err("discovery cannot complete");
|
||||
|
||||
assert!(matches!(error, GrabError::IndexerDiscoveryTimeout(_)));
|
||||
assert!(fake.torrents().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indexer_links_and_magnets_both_reach_transmission() {
|
||||
assert_eq!(
|
||||
|
||||
Reference in New Issue
Block a user