yolo/arr
3
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: RSS sync worker (#91)
ci / web (push) Successful in 26s
Details
e2e / e2e (push) Successful in 53s
Details
ci / rust (push) Successful in 2m13s
Details

Browse Source
This commit was merged in pull request #91.
This commit is contained in:
naps62-yolo (agent)
2026-08-23 01:04:15 +01:00
parent e01505bd53
commit 225be33b55
11 changed files with 1285 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/arr-daemon/src/grab.rs
+95 -103
View File
@@ -16,7 +16,7 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::time::{SystemTime, UNIX_EPOCH};
use arr_core::policy::{evaluate, Candidate};
use arr_core::{score::score, Language, Policy, TitleOverrides, Verdict};
@@ -25,6 +25,7 @@ use arr_dl::{AddTorrent, TorrentSource, TransmissionClient};
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
use arr_meta::TmdbClient;
use crate::indexers::{DiscoveryError, IndexerDirectory};
use crate::reconcile::{Action, ActionFuture, Outcome};
/// How many titles one tick may search. The reconcile lane has a 25 s budget
@@ -33,16 +34,6 @@ use crate::reconcile::{Action, ActionFuture, Outcome};
/// and release-date gating are issue #26.
const MOVIES_PER_TICK: i64 = 5;
/// How long a discovered indexer list is reused. Prowlarr enumerates one
/// indexer per call plus a `t=caps` probe each, all sequential, so paying for
/// it every 30 s would leave the tick no room to search.
const INDEXER_CACHE_TTL: Duration = Duration::from_mins(15);
/// The share of the reconcile lane's 25 s budget discovery may spend. One
/// unresponsive tracker must not cancel the whole action before a single
/// targeted search has run; past this the last known list is used instead.
const INDEXER_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(8);
/// Seeding obligations, per tracker in principle (§7.3) and per install in
/// practice until issue #25 gives them a home. Both are set on the torrent at
/// add time and enforced by Transmission.
@@ -79,8 +70,8 @@ pub enum GrabError {
Database(#[from] sqlx::Error),
#[error("policy: {0}")]
Policy(#[from] arr_db::PolicyError),
#[error("prowlarr: {0}")]
Prowlarr(#[from] arr_indexer::Error),
#[error("{0}")]
Discovery(#[from] DiscoveryError),
#[error("TMDB: {0}")]
Metadata(#[from] arr_meta::Error),
#[error("movie {0} has an invalid TMDB id")]
@@ -92,15 +83,6 @@ pub enum GrabError {
name: String,
source: serde_json::Error,
},
#[error("indexer discovery timed out after {0:?} and none is known yet")]
IndexerDiscoveryTimeout(Duration),
}
/// The searchable indexer ids, and when they were last discovered.
#[derive(Debug, Default)]
struct IndexerCache {
ids: Vec<i64>,
refreshed_at: Option<Instant>,
}
/// Sends the best eligible release for every wanted movie that has neither a
@@ -108,14 +90,9 @@ struct IndexerCache {
#[derive(Debug)]
pub struct GrabAction {
prowlarr: ProwlarrClient,
transmission: TransmissionClient,
download_dir: PathBuf,
seeding: SeedingRules,
grabber: Grabber,
tmdb: Option<Arc<TmdbClient>>,
indexers: tokio::sync::RwLock<IndexerCache>,
/// [`INDEXER_DISCOVERY_TIMEOUT`], overridden by tests that cannot wait
/// out the real one. Mirrors `ReconcileLoop`'s action timeout override.
discovery_timeout: Duration,
indexers: IndexerDirectory,
}
impl GrabAction {
@@ -127,13 +104,10 @@ impl GrabAction {
seeding: SeedingRules,
) -> Self {
Self {
indexers: IndexerDirectory::new(prowlarr.clone()),
prowlarr,
transmission,
download_dir,
seeding,
grabber: Grabber::new(transmission, download_dir, seeding),
tmdb: None,
indexers: tokio::sync::RwLock::new(IndexerCache::default()),
discovery_timeout: INDEXER_DISCOVERY_TIMEOUT,
}
}
@@ -165,7 +139,7 @@ impl GrabAction {
if !search_due(&movie) {
continue;
}
let searchable = self.searchable_indexers().await?;
let searchable = self.indexers.searchable().await?;
if searchable.is_empty() {
tracing::warn!("no indexer advertises a text search; nothing can be grabbed");
return Ok(outcomes);
@@ -290,6 +264,7 @@ impl GrabAction {
}
let torrents: HashMap<String, f64> = self
.grabber
.transmission
.list_torrents()
.await?
@@ -327,51 +302,6 @@ impl GrabAction {
Ok(outcomes)
}
/// The indexers that accept a text search, cached across ticks.
///
/// Discovery is bounded and its result is reused, because Prowlarr probes
/// capabilities one indexer at a time: a single slow tracker would
/// otherwise burn the whole reconcile budget before any search runs. A
/// refresh that fails or times out keeps the previous list rather than
/// stopping the tick, and only an empty cache turns that into an error.
async fn searchable_indexers(&self) -> Result<Vec<i64>, GrabError> {
{
let cache = self.indexers.read().await;
if cache
.refreshed_at
.is_some_and(|at| at.elapsed() < INDEXER_CACHE_TTL)
{
return Ok(cache.ids.clone());
}
}
let discovered =
tokio::time::timeout(self.discovery_timeout, self.prowlarr.indexers()).await;
let mut cache = self.indexers.write().await;
match discovered {
Ok(Ok(indexers)) => {
cache.ids = indexers
.iter()
.filter(|indexer| indexer.capabilities.search.available)
.map(|indexer| indexer.id)
.collect();
cache.refreshed_at = Some(Instant::now());
}
Ok(Err(error)) if cache.ids.is_empty() => return Err(error.into()),
Err(_) if cache.ids.is_empty() => {
return Err(GrabError::IndexerDiscoveryTimeout(self.discovery_timeout))
}
Ok(Err(error)) => {
tracing::warn!(%error, "indexer discovery failed; using the last known list");
}
Err(_) => tracing::warn!(
timeout_seconds = self.discovery_timeout.as_secs(),
"indexer discovery timed out; using the last known list"
),
}
Ok(cache.ids.clone())
}
/// Search every indexer for one title, cache each candidate with its
/// verdict and score (§9.3), and return the eligible ones best first.
///
@@ -475,9 +405,67 @@ impl GrabAction {
return Ok(None);
};
self.send_winner(database, movie, &loaded, &blacklist, winner)
self.grabber
.send_winner(
database,
&GrabTarget {
movie_id: movie.id,
title: &movie.title,
counts_as_attempt: true,
},
&loaded,
&blacklist,
winner,
)
.await
}
}
/// Sending a chosen release to Transmission and recording the grab.
///
/// Targeted search and RSS (§6.2) differ in how a title is chosen and in
/// whether a failed attempt counts toward a backoff; from the winning
/// release onward they are the same writes, so they share this.
#[derive(Debug)]
pub(crate) struct Grabber {
transmission: TransmissionClient,
download_dir: PathBuf,
seeding: SeedingRules,
}
/// The title a winning release is being grabbed for.
#[derive(Debug)]
pub(crate) struct GrabTarget<'a> {
pub(crate) movie_id: i64,
pub(crate) title: &'a str,
/// Whether a grab that does not complete counts toward the targeted
/// search backoff (§6.2). RSS never backs off, so it passes `false`.
pub(crate) counts_as_attempt: bool,
}
impl Grabber {
pub(crate) fn new(
transmission: TransmissionClient,
download_dir: PathBuf,
seeding: SeedingRules,
) -> Self {
Self {
transmission,
download_dir,
seeding,
}
}
async fn record_attempt(
&self,
database: &Db,
target: &GrabTarget<'_>,
) -> Result<(), GrabError> {
if target.counts_as_attempt {
record_search(database, target.movie_id).await?;
}
Ok(())
}
/// Add the winning release to Transmission and record the grab.
///
@@ -485,10 +473,10 @@ impl GrabAction {
/// not a completed grab — a Transmission error or a blacklisted-infohash
/// drop must not leave the same release to repeat next tick with no
/// backoff.
async fn send_winner(
pub(crate) async fn send_winner(
&self,
database: &Db,
movie: &PendingMovie,
target: &GrabTarget<'_>,
loaded: &MoviePolicy,
blacklist: &Blacklist,
winner: Eligible,
@@ -507,7 +495,7 @@ impl GrabAction {
{
Ok(added) => added,
Err(error) => {
record_search(database, movie.id).await?;
self.record_attempt(database, target).await?;
return Err(error.into());
}
};
@@ -517,8 +505,8 @@ impl GrabAction {
// Transmission has fetched it, so the same blacklisted torrent can
// reach here under a new name.
if blacklist.blocks_infohash(&infohash) {
record_search(database, movie.id).await?;
self.drop_blacklisted_torrent(database, movie, &winner, &added)
self.record_attempt(database, target).await?;
self.drop_blacklisted_torrent(database, target, &winner, &added)
.await?;
return Ok(None);
}
@@ -531,7 +519,7 @@ impl GrabAction {
ON CONFLICT (infohash) DO NOTHING
RETURNING id AS "id!: i64""#,
winner.id,
movie.id,
target.movie_id,
infohash
)
.fetch_optional(database.pool())
@@ -540,14 +528,14 @@ impl GrabAction {
"UPDATE movies SET state = 'downloading',
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
WHERE id = ?",
movie.id
target.movie_id
)
.execute(database.pool())
.await?;
let Some(inserted) = inserted else {
tracing::info!(
movie_id = movie.id,
movie_id = target.movie_id,
infohash,
"grab already recorded for this torrent"
);
@@ -555,8 +543,8 @@ impl GrabAction {
};
tracing::info!(
movie_id = movie.id,
title = movie.title,
movie_id = target.movie_id,
title = target.title,
release = winner.name,
score = winner.score,
infohash,
@@ -564,7 +552,7 @@ impl GrabAction {
"grabbed"
);
Ok(Some(Outcome::new(
format!("movie {} wanted with no file", movie.id),
format!("movie {} wanted with no file", target.movie_id),
format!("grabbed {} as grab {}", winner.name, inserted.id),
)))
}
@@ -577,7 +565,7 @@ impl GrabAction {
async fn drop_blacklisted_torrent(
&self,
database: &Db,
movie: &PendingMovie,
target: &GrabTarget<'_>,
winner: &Eligible,
added: &arr_dl::AddedTorrent,
) -> Result<(), GrabError> {
@@ -604,7 +592,7 @@ impl GrabAction {
// The earlier grab's torrent, still working off its seeding
// obligation (§7.3). Nothing here deletes a torrent.
tracing::warn!(
movie_id = movie.id,
movie_id = target.movie_id,
release = release_name,
infohash = added.hash,
"blacklisted torrent re-listed under a new name; left seeding"
@@ -614,7 +602,7 @@ impl GrabAction {
// obligation and has nothing on disk worth keeping.
self.transmission.remove_torrent(added.id, true).await?;
tracing::warn!(
movie_id = movie.id,
movie_id = target.movie_id,
release = release_name,
infohash = added.hash,
"blacklisted torrent re-listed under a new name; removed"
@@ -652,13 +640,13 @@ struct PendingMovie {
/// The eligible view of a stored release, ranked for selection.
#[derive(Debug, Clone)]
struct Eligible {
pub(crate) struct Eligible {
id: i64,
indexer_id: i64,
guid: String,
name: String,
pub(crate) indexer_id: i64,
pub(crate) guid: String,
pub(crate) name: String,
download_url: String,
score: i64,
pub(crate) score: i64,
}
/// The gap, straight out of the domain rows (§8).
@@ -759,7 +747,7 @@ fn is_digitally_released(digital_release: Option<&str>) -> bool {
/// trigger (§6.3: "including by RSS") and so the stored row says why — a
/// blacklisted release is rejected under the `blacklisted` rule, which is
/// what stops §9.3's manual view from offering it as a clean match.
async fn store_release(
pub(crate) async fn store_release(
database: &Db,
movie_id: i64,
release: &SearchRelease,
@@ -919,6 +907,7 @@ fn rfc3339(time: SystemTime) -> Option<String> {
#[allow(clippy::unwrap_used)]
mod tests {
use std::sync::{Arc, Mutex};
use std::time::Duration;
use serde_json::{json, Value};
use wiremock::matchers::{method, path, query_param};
@@ -1697,14 +1686,17 @@ mod tests {
let (downloader, fake) = transmission().await;
let mut action = action(&indexer, &downloader);
action.discovery_timeout = Duration::from_millis(50);
action.indexers.discovery_timeout = Duration::from_millis(50);
let error = action
.tick(&database)
.await
.expect_err("discovery cannot complete");
assert!(matches!(error, GrabError::IndexerDiscoveryTimeout(_)));
assert!(matches!(
error,
GrabError::Discovery(DiscoveryError::Timeout(_))
));
assert!(fake.torrents().is_empty());
}