feat: RSS sync worker (#91)
This commit was merged in pull request #91.
This commit is contained in:
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT original_language FROM movies WHERE id = ?",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "original_language",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Text"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
true
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "7fa29678b941e5eb680db2dcb11ed1912b950a84d1ce7eaa406339b024aac82d"
|
||||||
|
}
|
||||||
+38
@@ -0,0 +1,38 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "\n SELECT m.id AS \"id!: i64\",\n m.tmdb_id AS \"tmdb_id!: i64\",\n m.title AS \"title!: String\",\n m.year\n FROM movies m\n WHERE m.wanted = 1\n AND NOT EXISTS (\n SELECT 1 FROM media_files f\n WHERE f.owner_kind = 'movie' AND f.owner_id = m.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM grabs g\n WHERE g.target_kind = 'movie' AND g.target_id = m.id\n AND g.state IN ('sent', 'downloaded', 'imported')\n )\n ORDER BY m.id\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "id!: i64",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tmdb_id!: i64",
|
||||||
|
"ordinal": 1,
|
||||||
|
"type_info": "Integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "title!: String",
|
||||||
|
"ordinal": 2,
|
||||||
|
"type_info": "Text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "year",
|
||||||
|
"ordinal": 3,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 0
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
true
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "d318af0005db414990095d0d86e312f26fd0e57018ba6a4667d53b590080d4a7"
|
||||||
|
}
|
||||||
@@ -831,6 +831,7 @@ mod tests {
|
|||||||
seeders: Some(8),
|
seeders: Some(8),
|
||||||
publish_date: None,
|
publish_date: None,
|
||||||
download_url: "https://tracker/release".into(),
|
download_url: "https://tracker/release".into(),
|
||||||
|
tmdb_id: None,
|
||||||
};
|
};
|
||||||
let parsed = arr_parse::parse(&release.name);
|
let parsed = arr_parse::parse(&release.name);
|
||||||
let core_score = score(&policy, Candidate::PreGrab(&parsed), 0, 8);
|
let core_score = score(&policy, Candidate::PreGrab(&parsed), 0, 8);
|
||||||
|
|||||||
@@ -6,12 +6,14 @@ use std::{collections::BTreeMap, fmt, path::PathBuf, time::SystemTime};
|
|||||||
|
|
||||||
pub mod lang;
|
pub mod lang;
|
||||||
pub mod layout;
|
pub mod layout;
|
||||||
|
pub mod matching;
|
||||||
pub mod policy;
|
pub mod policy;
|
||||||
pub mod score;
|
pub mod score;
|
||||||
pub mod status;
|
pub mod status;
|
||||||
pub mod tracking;
|
pub mod tracking;
|
||||||
|
|
||||||
pub use arr_parse::NameClaims as ParsedRelease;
|
pub use arr_parse::NameClaims as ParsedRelease;
|
||||||
|
pub use matching::{match_movie, MatchKind, MovieMatch, WantedMovie};
|
||||||
pub use score::{Score, ScoreWeights};
|
pub use score::{Score, ScoreWeights};
|
||||||
pub use status::{derive_series_status, SeriesStatus};
|
pub use status::{derive_series_status, SeriesStatus};
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,278 @@
|
|||||||
|
//! Matching an RSS result against the wanted list (`DESIGN.md` §6.2).
|
||||||
|
//!
|
||||||
|
//! RSS is an empty-query feed: every item is offered to the whole wanted list
|
||||||
|
//! and nothing about the query says which title an item is for. That makes a
|
||||||
|
//! false positive expensive — it grabs the wrong film — and a false negative
|
||||||
|
//! nearly free, because the feed is read again ten minutes later and the
|
||||||
|
//! targeted search still runs. So every rule here is deliberately strict:
|
||||||
|
//!
|
||||||
|
//! - an ID the tracker itself supplied beats anything read off the name, and
|
||||||
|
//! an ID that matches nothing wanted ends the comparison rather than
|
||||||
|
//! falling back to the title;
|
||||||
|
//! - a title match needs the years to agree, and a wanted title with a known
|
||||||
|
//! year is never matched by a release that does not state one;
|
||||||
|
//! - anything that matches two wanted titles matches neither.
|
||||||
|
|
||||||
|
use crate::MovieId;
|
||||||
|
use arr_parse::NameClaims;
|
||||||
|
|
||||||
|
/// A wanted title, in the shape matching needs.
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub struct WantedMovie {
|
||||||
|
pub id: MovieId,
|
||||||
|
/// TMDB id, compared against the one the tracker attached to the item.
|
||||||
|
pub tmdb_id: Option<u32>,
|
||||||
|
pub title: String,
|
||||||
|
pub year: Option<u16>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What tied a release to a wanted title.
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
pub enum MatchKind {
|
||||||
|
/// The tracker supplied a TMDB id and it is one of ours.
|
||||||
|
TmdbId,
|
||||||
|
/// The parsed release name and the wanted title agree, years included.
|
||||||
|
TitleAndYear,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One wanted title an RSS item was matched to.
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
pub struct MovieMatch {
|
||||||
|
pub movie: MovieId,
|
||||||
|
pub kind: MatchKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The wanted title an RSS item belongs to, or `None` when nothing matches
|
||||||
|
/// conservatively.
|
||||||
|
///
|
||||||
|
/// `tmdb_id` is what the indexer attached to the item, not anything inferred
|
||||||
|
/// from the name; `claims` is the parsed release name.
|
||||||
|
#[must_use]
|
||||||
|
pub fn match_movie(
|
||||||
|
wanted: &[WantedMovie],
|
||||||
|
tmdb_id: Option<u32>,
|
||||||
|
claims: &NameClaims,
|
||||||
|
) -> Option<MovieMatch> {
|
||||||
|
if let Some(tmdb_id) = tmdb_id {
|
||||||
|
return unique(
|
||||||
|
wanted
|
||||||
|
.iter()
|
||||||
|
.filter(|movie| movie.tmdb_id == Some(tmdb_id))
|
||||||
|
.map(|movie| MovieMatch {
|
||||||
|
movie: movie.id,
|
||||||
|
kind: MatchKind::TmdbId,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A season or episode tag makes this a TV release, whatever the title
|
||||||
|
// says. Movies are the only thing RSS grabs today (§13, phase 4).
|
||||||
|
if claims.episode.is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let title = match_key(claims.title.as_deref()?);
|
||||||
|
if title.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
unique(
|
||||||
|
wanted
|
||||||
|
.iter()
|
||||||
|
.filter(|movie| {
|
||||||
|
match_key(&movie.title) == title && years_agree(movie.year, claims.year)
|
||||||
|
})
|
||||||
|
.map(|movie| MovieMatch {
|
||||||
|
movie: movie.id,
|
||||||
|
kind: MatchKind::TitleAndYear,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A release with no year matches only a title that has none either: films
|
||||||
|
/// share titles across decades, and the remake is not the one that is wanted.
|
||||||
|
fn years_agree(wanted: Option<u16>, release: Option<u16>) -> bool {
|
||||||
|
match (wanted, release) {
|
||||||
|
(Some(wanted), Some(release)) => wanted == release,
|
||||||
|
(None, None) => true,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Two matches are an ambiguity, and an ambiguity is not a match.
|
||||||
|
fn unique(mut matches: impl Iterator<Item = MovieMatch>) -> Option<MovieMatch> {
|
||||||
|
let first = matches.next()?;
|
||||||
|
matches.next().is_none().then_some(first)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The form two titles are compared in: lower case, accents folded to ASCII,
|
||||||
|
/// `&` spelled out, and every run of anything else one space.
|
||||||
|
///
|
||||||
|
/// This absorbs the separators and punctuation a release name loses ("Dune:
|
||||||
|
/// Part Two" against `Dune.Part.Two`) without absorbing a word, so a longer
|
||||||
|
/// or shorter title stays a different title.
|
||||||
|
fn match_key(title: &str) -> String {
|
||||||
|
let mut key = String::with_capacity(title.len());
|
||||||
|
let mut gap = false;
|
||||||
|
for character in title.chars() {
|
||||||
|
if let Some(folded) = fold(character) {
|
||||||
|
separate(&mut key, &mut gap);
|
||||||
|
key.push_str(folded);
|
||||||
|
} else if character.is_alphanumeric() {
|
||||||
|
separate(&mut key, &mut gap);
|
||||||
|
key.extend(character.to_lowercase());
|
||||||
|
} else {
|
||||||
|
gap = !key.is_empty();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
key
|
||||||
|
}
|
||||||
|
|
||||||
|
fn separate(key: &mut String, gap: &mut bool) {
|
||||||
|
if *gap {
|
||||||
|
key.push(' ');
|
||||||
|
}
|
||||||
|
*gap = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ASCII a letter is written as when a tracker cannot spell the accent,
|
||||||
|
/// and the word an `&` is spelled out as. `None` leaves the character to
|
||||||
|
/// [`match_key`], which keeps any other letter or digit as it is.
|
||||||
|
fn fold(character: char) -> Option<&'static str> {
|
||||||
|
Some(match character {
|
||||||
|
'&' => "and",
|
||||||
|
'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' => "a",
|
||||||
|
'ç' | 'Ç' => "c",
|
||||||
|
'è' | 'é' | 'ê' | 'ë' | 'È' | 'É' | 'Ê' | 'Ë' => "e",
|
||||||
|
'ì' | 'í' | 'î' | 'ï' | 'Ì' | 'Í' | 'Î' | 'Ï' => "i",
|
||||||
|
'ñ' | 'Ñ' => "n",
|
||||||
|
'ò' | 'ó' | 'ô' | 'õ' | 'ö' | 'ø' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | 'Ø' => "o",
|
||||||
|
'ù' | 'ú' | 'û' | 'ü' | 'Ù' | 'Ú' | 'Û' | 'Ü' => "u",
|
||||||
|
'ý' | 'ÿ' | 'Ý' => "y",
|
||||||
|
'ß' => "ss",
|
||||||
|
'æ' | 'Æ' => "ae",
|
||||||
|
_ => return None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{match_key, match_movie, MatchKind, MovieMatch, WantedMovie};
|
||||||
|
use crate::MovieId;
|
||||||
|
|
||||||
|
fn wanted() -> Vec<WantedMovie> {
|
||||||
|
vec![
|
||||||
|
WantedMovie {
|
||||||
|
id: MovieId(1),
|
||||||
|
tmdb_id: Some(693_134),
|
||||||
|
title: "Dune: Part Two".to_owned(),
|
||||||
|
year: Some(2024),
|
||||||
|
},
|
||||||
|
WantedMovie {
|
||||||
|
id: MovieId(2),
|
||||||
|
tmdb_id: Some(438_631),
|
||||||
|
title: "Dune".to_owned(),
|
||||||
|
year: Some(2021),
|
||||||
|
},
|
||||||
|
WantedMovie {
|
||||||
|
id: MovieId(3),
|
||||||
|
tmdb_id: Some(194),
|
||||||
|
title: "Amélie".to_owned(),
|
||||||
|
year: Some(2001),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matched(name: &str, tmdb_id: Option<u32>) -> Option<MovieMatch> {
|
||||||
|
match_movie(&wanted(), tmdb_id, &arr_parse::parse(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_supplied_id_matches_whatever_the_name_says() {
|
||||||
|
assert_eq!(
|
||||||
|
matched("Some.Scene.Name.2024.2160p.WEB-DL", Some(693_134)),
|
||||||
|
Some(MovieMatch {
|
||||||
|
movie: MovieId(1),
|
||||||
|
kind: MatchKind::TmdbId,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The tracker said which film this is and it is not one of ours. Falling
|
||||||
|
/// back to the title here is how a wrong film gets grabbed.
|
||||||
|
#[test]
|
||||||
|
fn a_supplied_id_that_is_not_wanted_ends_the_comparison() {
|
||||||
|
assert_eq!(matched("Dune.Part.Two.2024.2160p.WEB-DL", Some(11)), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn separators_punctuation_and_accents_do_not_block_a_title_match() {
|
||||||
|
assert_eq!(
|
||||||
|
matched("Dune.Part.Two.2024.2160p.WEB-DL.DDP5.1.Atmos-GROUP", None),
|
||||||
|
Some(MovieMatch {
|
||||||
|
movie: MovieId(1),
|
||||||
|
kind: MatchKind::TitleAndYear,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
matched("Amelie.2001.1080p.BluRay.x264-GROUP", None),
|
||||||
|
Some(MovieMatch {
|
||||||
|
movie: MovieId(3),
|
||||||
|
kind: MatchKind::TitleAndYear,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The near miss: one title is a prefix of the other, same year band,
|
||||||
|
/// same franchise. Nothing here may match.
|
||||||
|
#[test]
|
||||||
|
fn a_near_miss_does_not_match() {
|
||||||
|
assert_eq!(matched("Dune.2024.2160p.WEB-DL-GROUP", None), None);
|
||||||
|
assert_eq!(matched("Dune.Part.One.2021.1080p.WEB-DL", None), None);
|
||||||
|
assert_eq!(matched("Dune.Part.Two.2023.1080p.WEB-DL", None), None);
|
||||||
|
assert_eq!(matched("Dune.Prophecy.2024.1080p.WEB-DL", None), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_release_with_no_year_is_not_matched_to_a_title_that_has_one() {
|
||||||
|
assert_eq!(matched("Dune.Part.Two.2160p.WEB-DL", None), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_episode_tag_is_never_a_movie() {
|
||||||
|
let wanted = vec![WantedMovie {
|
||||||
|
id: MovieId(4),
|
||||||
|
tmdb_id: Some(1),
|
||||||
|
title: "Fallout".to_owned(),
|
||||||
|
year: Some(2024),
|
||||||
|
}];
|
||||||
|
let claims = arr_parse::parse("Fallout.2024.S01E03.1080p.WEB-DL");
|
||||||
|
assert_eq!(match_movie(&wanted, None, &claims), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn two_wanted_titles_that_both_match_are_an_ambiguity() {
|
||||||
|
let wanted = vec![
|
||||||
|
WantedMovie {
|
||||||
|
id: MovieId(5),
|
||||||
|
tmdb_id: Some(1),
|
||||||
|
title: "The Thing".to_owned(),
|
||||||
|
year: Some(1982),
|
||||||
|
},
|
||||||
|
WantedMovie {
|
||||||
|
id: MovieId(6),
|
||||||
|
tmdb_id: Some(2),
|
||||||
|
title: "The Thing".to_owned(),
|
||||||
|
year: Some(1982),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let claims = arr_parse::parse("The.Thing.1982.1080p.BluRay");
|
||||||
|
assert_eq!(match_movie(&wanted, None, &claims), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ampersands_and_spelling_agree() {
|
||||||
|
assert_eq!(match_key("Fast & Furious"), "fast and furious");
|
||||||
|
assert_eq!(match_key("Fast and Furious"), "fast and furious");
|
||||||
|
assert_eq!(match_key(" Dune: Part Two "), "dune part two");
|
||||||
|
}
|
||||||
|
}
|
||||||
+95
-103
@@ -16,7 +16,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use arr_core::policy::{evaluate, Candidate};
|
use arr_core::policy::{evaluate, Candidate};
|
||||||
use arr_core::{score::score, Language, Policy, TitleOverrides, Verdict};
|
use arr_core::{score::score, Language, Policy, TitleOverrides, Verdict};
|
||||||
@@ -25,6 +25,7 @@ use arr_dl::{AddTorrent, TorrentSource, TransmissionClient};
|
|||||||
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
|
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
|
||||||
use arr_meta::TmdbClient;
|
use arr_meta::TmdbClient;
|
||||||
|
|
||||||
|
use crate::indexers::{DiscoveryError, IndexerDirectory};
|
||||||
use crate::reconcile::{Action, ActionFuture, Outcome};
|
use crate::reconcile::{Action, ActionFuture, Outcome};
|
||||||
|
|
||||||
/// How many titles one tick may search. The reconcile lane has a 25 s budget
|
/// How many titles one tick may search. The reconcile lane has a 25 s budget
|
||||||
@@ -33,16 +34,6 @@ use crate::reconcile::{Action, ActionFuture, Outcome};
|
|||||||
/// and release-date gating are issue #26.
|
/// and release-date gating are issue #26.
|
||||||
const MOVIES_PER_TICK: i64 = 5;
|
const MOVIES_PER_TICK: i64 = 5;
|
||||||
|
|
||||||
/// How long a discovered indexer list is reused. Prowlarr enumerates one
|
|
||||||
/// indexer per call plus a `t=caps` probe each, all sequential, so paying for
|
|
||||||
/// it every 30 s would leave the tick no room to search.
|
|
||||||
const INDEXER_CACHE_TTL: Duration = Duration::from_mins(15);
|
|
||||||
|
|
||||||
/// The share of the reconcile lane's 25 s budget discovery may spend. One
|
|
||||||
/// unresponsive tracker must not cancel the whole action before a single
|
|
||||||
/// targeted search has run; past this the last known list is used instead.
|
|
||||||
const INDEXER_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(8);
|
|
||||||
|
|
||||||
/// Seeding obligations, per tracker in principle (§7.3) and per install in
|
/// Seeding obligations, per tracker in principle (§7.3) and per install in
|
||||||
/// practice until issue #25 gives them a home. Both are set on the torrent at
|
/// practice until issue #25 gives them a home. Both are set on the torrent at
|
||||||
/// add time and enforced by Transmission.
|
/// add time and enforced by Transmission.
|
||||||
@@ -79,8 +70,8 @@ pub enum GrabError {
|
|||||||
Database(#[from] sqlx::Error),
|
Database(#[from] sqlx::Error),
|
||||||
#[error("policy: {0}")]
|
#[error("policy: {0}")]
|
||||||
Policy(#[from] arr_db::PolicyError),
|
Policy(#[from] arr_db::PolicyError),
|
||||||
#[error("prowlarr: {0}")]
|
#[error("{0}")]
|
||||||
Prowlarr(#[from] arr_indexer::Error),
|
Discovery(#[from] DiscoveryError),
|
||||||
#[error("TMDB: {0}")]
|
#[error("TMDB: {0}")]
|
||||||
Metadata(#[from] arr_meta::Error),
|
Metadata(#[from] arr_meta::Error),
|
||||||
#[error("movie {0} has an invalid TMDB id")]
|
#[error("movie {0} has an invalid TMDB id")]
|
||||||
@@ -92,15 +83,6 @@ pub enum GrabError {
|
|||||||
name: String,
|
name: String,
|
||||||
source: serde_json::Error,
|
source: serde_json::Error,
|
||||||
},
|
},
|
||||||
#[error("indexer discovery timed out after {0:?} and none is known yet")]
|
|
||||||
IndexerDiscoveryTimeout(Duration),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The searchable indexer ids, and when they were last discovered.
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
struct IndexerCache {
|
|
||||||
ids: Vec<i64>,
|
|
||||||
refreshed_at: Option<Instant>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sends the best eligible release for every wanted movie that has neither a
|
/// Sends the best eligible release for every wanted movie that has neither a
|
||||||
@@ -108,14 +90,9 @@ struct IndexerCache {
|
|||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct GrabAction {
|
pub struct GrabAction {
|
||||||
prowlarr: ProwlarrClient,
|
prowlarr: ProwlarrClient,
|
||||||
transmission: TransmissionClient,
|
grabber: Grabber,
|
||||||
download_dir: PathBuf,
|
|
||||||
seeding: SeedingRules,
|
|
||||||
tmdb: Option<Arc<TmdbClient>>,
|
tmdb: Option<Arc<TmdbClient>>,
|
||||||
indexers: tokio::sync::RwLock<IndexerCache>,
|
indexers: IndexerDirectory,
|
||||||
/// [`INDEXER_DISCOVERY_TIMEOUT`], overridden by tests that cannot wait
|
|
||||||
/// out the real one. Mirrors `ReconcileLoop`'s action timeout override.
|
|
||||||
discovery_timeout: Duration,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl GrabAction {
|
impl GrabAction {
|
||||||
@@ -127,13 +104,10 @@ impl GrabAction {
|
|||||||
seeding: SeedingRules,
|
seeding: SeedingRules,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
indexers: IndexerDirectory::new(prowlarr.clone()),
|
||||||
prowlarr,
|
prowlarr,
|
||||||
transmission,
|
grabber: Grabber::new(transmission, download_dir, seeding),
|
||||||
download_dir,
|
|
||||||
seeding,
|
|
||||||
tmdb: None,
|
tmdb: None,
|
||||||
indexers: tokio::sync::RwLock::new(IndexerCache::default()),
|
|
||||||
discovery_timeout: INDEXER_DISCOVERY_TIMEOUT,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,7 +139,7 @@ impl GrabAction {
|
|||||||
if !search_due(&movie) {
|
if !search_due(&movie) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let searchable = self.searchable_indexers().await?;
|
let searchable = self.indexers.searchable().await?;
|
||||||
if searchable.is_empty() {
|
if searchable.is_empty() {
|
||||||
tracing::warn!("no indexer advertises a text search; nothing can be grabbed");
|
tracing::warn!("no indexer advertises a text search; nothing can be grabbed");
|
||||||
return Ok(outcomes);
|
return Ok(outcomes);
|
||||||
@@ -290,6 +264,7 @@ impl GrabAction {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let torrents: HashMap<String, f64> = self
|
let torrents: HashMap<String, f64> = self
|
||||||
|
.grabber
|
||||||
.transmission
|
.transmission
|
||||||
.list_torrents()
|
.list_torrents()
|
||||||
.await?
|
.await?
|
||||||
@@ -327,51 +302,6 @@ impl GrabAction {
|
|||||||
Ok(outcomes)
|
Ok(outcomes)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The indexers that accept a text search, cached across ticks.
|
|
||||||
///
|
|
||||||
/// Discovery is bounded and its result is reused, because Prowlarr probes
|
|
||||||
/// capabilities one indexer at a time: a single slow tracker would
|
|
||||||
/// otherwise burn the whole reconcile budget before any search runs. A
|
|
||||||
/// refresh that fails or times out keeps the previous list rather than
|
|
||||||
/// stopping the tick, and only an empty cache turns that into an error.
|
|
||||||
async fn searchable_indexers(&self) -> Result<Vec<i64>, GrabError> {
|
|
||||||
{
|
|
||||||
let cache = self.indexers.read().await;
|
|
||||||
if cache
|
|
||||||
.refreshed_at
|
|
||||||
.is_some_and(|at| at.elapsed() < INDEXER_CACHE_TTL)
|
|
||||||
{
|
|
||||||
return Ok(cache.ids.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let discovered =
|
|
||||||
tokio::time::timeout(self.discovery_timeout, self.prowlarr.indexers()).await;
|
|
||||||
let mut cache = self.indexers.write().await;
|
|
||||||
match discovered {
|
|
||||||
Ok(Ok(indexers)) => {
|
|
||||||
cache.ids = indexers
|
|
||||||
.iter()
|
|
||||||
.filter(|indexer| indexer.capabilities.search.available)
|
|
||||||
.map(|indexer| indexer.id)
|
|
||||||
.collect();
|
|
||||||
cache.refreshed_at = Some(Instant::now());
|
|
||||||
}
|
|
||||||
Ok(Err(error)) if cache.ids.is_empty() => return Err(error.into()),
|
|
||||||
Err(_) if cache.ids.is_empty() => {
|
|
||||||
return Err(GrabError::IndexerDiscoveryTimeout(self.discovery_timeout))
|
|
||||||
}
|
|
||||||
Ok(Err(error)) => {
|
|
||||||
tracing::warn!(%error, "indexer discovery failed; using the last known list");
|
|
||||||
}
|
|
||||||
Err(_) => tracing::warn!(
|
|
||||||
timeout_seconds = self.discovery_timeout.as_secs(),
|
|
||||||
"indexer discovery timed out; using the last known list"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
Ok(cache.ids.clone())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Search every indexer for one title, cache each candidate with its
|
/// Search every indexer for one title, cache each candidate with its
|
||||||
/// verdict and score (§9.3), and return the eligible ones best first.
|
/// verdict and score (§9.3), and return the eligible ones best first.
|
||||||
///
|
///
|
||||||
@@ -475,9 +405,67 @@ impl GrabAction {
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
self.send_winner(database, movie, &loaded, &blacklist, winner)
|
self.grabber
|
||||||
|
.send_winner(
|
||||||
|
database,
|
||||||
|
&GrabTarget {
|
||||||
|
movie_id: movie.id,
|
||||||
|
title: &movie.title,
|
||||||
|
counts_as_attempt: true,
|
||||||
|
},
|
||||||
|
&loaded,
|
||||||
|
&blacklist,
|
||||||
|
winner,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sending a chosen release to Transmission and recording the grab.
|
||||||
|
///
|
||||||
|
/// Targeted search and RSS (§6.2) differ in how a title is chosen and in
|
||||||
|
/// whether a failed attempt counts toward a backoff; from the winning
|
||||||
|
/// release onward they are the same writes, so they share this.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct Grabber {
|
||||||
|
transmission: TransmissionClient,
|
||||||
|
download_dir: PathBuf,
|
||||||
|
seeding: SeedingRules,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The title a winning release is being grabbed for.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct GrabTarget<'a> {
|
||||||
|
pub(crate) movie_id: i64,
|
||||||
|
pub(crate) title: &'a str,
|
||||||
|
/// Whether a grab that does not complete counts toward the targeted
|
||||||
|
/// search backoff (§6.2). RSS never backs off, so it passes `false`.
|
||||||
|
pub(crate) counts_as_attempt: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Grabber {
|
||||||
|
pub(crate) fn new(
|
||||||
|
transmission: TransmissionClient,
|
||||||
|
download_dir: PathBuf,
|
||||||
|
seeding: SeedingRules,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
transmission,
|
||||||
|
download_dir,
|
||||||
|
seeding,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn record_attempt(
|
||||||
|
&self,
|
||||||
|
database: &Db,
|
||||||
|
target: &GrabTarget<'_>,
|
||||||
|
) -> Result<(), GrabError> {
|
||||||
|
if target.counts_as_attempt {
|
||||||
|
record_search(database, target.movie_id).await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Add the winning release to Transmission and record the grab.
|
/// Add the winning release to Transmission and record the grab.
|
||||||
///
|
///
|
||||||
@@ -485,10 +473,10 @@ impl GrabAction {
|
|||||||
/// not a completed grab — a Transmission error or a blacklisted-infohash
|
/// not a completed grab — a Transmission error or a blacklisted-infohash
|
||||||
/// drop must not leave the same release to repeat next tick with no
|
/// drop must not leave the same release to repeat next tick with no
|
||||||
/// backoff.
|
/// backoff.
|
||||||
async fn send_winner(
|
pub(crate) async fn send_winner(
|
||||||
&self,
|
&self,
|
||||||
database: &Db,
|
database: &Db,
|
||||||
movie: &PendingMovie,
|
target: &GrabTarget<'_>,
|
||||||
loaded: &MoviePolicy,
|
loaded: &MoviePolicy,
|
||||||
blacklist: &Blacklist,
|
blacklist: &Blacklist,
|
||||||
winner: Eligible,
|
winner: Eligible,
|
||||||
@@ -507,7 +495,7 @@ impl GrabAction {
|
|||||||
{
|
{
|
||||||
Ok(added) => added,
|
Ok(added) => added,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
record_search(database, movie.id).await?;
|
self.record_attempt(database, target).await?;
|
||||||
return Err(error.into());
|
return Err(error.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -517,8 +505,8 @@ impl GrabAction {
|
|||||||
// Transmission has fetched it, so the same blacklisted torrent can
|
// Transmission has fetched it, so the same blacklisted torrent can
|
||||||
// reach here under a new name.
|
// reach here under a new name.
|
||||||
if blacklist.blocks_infohash(&infohash) {
|
if blacklist.blocks_infohash(&infohash) {
|
||||||
record_search(database, movie.id).await?;
|
self.record_attempt(database, target).await?;
|
||||||
self.drop_blacklisted_torrent(database, movie, &winner, &added)
|
self.drop_blacklisted_torrent(database, target, &winner, &added)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
@@ -531,7 +519,7 @@ impl GrabAction {
|
|||||||
ON CONFLICT (infohash) DO NOTHING
|
ON CONFLICT (infohash) DO NOTHING
|
||||||
RETURNING id AS "id!: i64""#,
|
RETURNING id AS "id!: i64""#,
|
||||||
winner.id,
|
winner.id,
|
||||||
movie.id,
|
target.movie_id,
|
||||||
infohash
|
infohash
|
||||||
)
|
)
|
||||||
.fetch_optional(database.pool())
|
.fetch_optional(database.pool())
|
||||||
@@ -540,14 +528,14 @@ impl GrabAction {
|
|||||||
"UPDATE movies SET state = 'downloading',
|
"UPDATE movies SET state = 'downloading',
|
||||||
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
||||||
WHERE id = ?",
|
WHERE id = ?",
|
||||||
movie.id
|
target.movie_id
|
||||||
)
|
)
|
||||||
.execute(database.pool())
|
.execute(database.pool())
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let Some(inserted) = inserted else {
|
let Some(inserted) = inserted else {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
movie_id = movie.id,
|
movie_id = target.movie_id,
|
||||||
infohash,
|
infohash,
|
||||||
"grab already recorded for this torrent"
|
"grab already recorded for this torrent"
|
||||||
);
|
);
|
||||||
@@ -555,8 +543,8 @@ impl GrabAction {
|
|||||||
};
|
};
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
movie_id = movie.id,
|
movie_id = target.movie_id,
|
||||||
title = movie.title,
|
title = target.title,
|
||||||
release = winner.name,
|
release = winner.name,
|
||||||
score = winner.score,
|
score = winner.score,
|
||||||
infohash,
|
infohash,
|
||||||
@@ -564,7 +552,7 @@ impl GrabAction {
|
|||||||
"grabbed"
|
"grabbed"
|
||||||
);
|
);
|
||||||
Ok(Some(Outcome::new(
|
Ok(Some(Outcome::new(
|
||||||
format!("movie {} wanted with no file", movie.id),
|
format!("movie {} wanted with no file", target.movie_id),
|
||||||
format!("grabbed {} as grab {}", winner.name, inserted.id),
|
format!("grabbed {} as grab {}", winner.name, inserted.id),
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
@@ -577,7 +565,7 @@ impl GrabAction {
|
|||||||
async fn drop_blacklisted_torrent(
|
async fn drop_blacklisted_torrent(
|
||||||
&self,
|
&self,
|
||||||
database: &Db,
|
database: &Db,
|
||||||
movie: &PendingMovie,
|
target: &GrabTarget<'_>,
|
||||||
winner: &Eligible,
|
winner: &Eligible,
|
||||||
added: &arr_dl::AddedTorrent,
|
added: &arr_dl::AddedTorrent,
|
||||||
) -> Result<(), GrabError> {
|
) -> Result<(), GrabError> {
|
||||||
@@ -604,7 +592,7 @@ impl GrabAction {
|
|||||||
// The earlier grab's torrent, still working off its seeding
|
// The earlier grab's torrent, still working off its seeding
|
||||||
// obligation (§7.3). Nothing here deletes a torrent.
|
// obligation (§7.3). Nothing here deletes a torrent.
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
movie_id = movie.id,
|
movie_id = target.movie_id,
|
||||||
release = release_name,
|
release = release_name,
|
||||||
infohash = added.hash,
|
infohash = added.hash,
|
||||||
"blacklisted torrent re-listed under a new name; left seeding"
|
"blacklisted torrent re-listed under a new name; left seeding"
|
||||||
@@ -614,7 +602,7 @@ impl GrabAction {
|
|||||||
// obligation and has nothing on disk worth keeping.
|
// obligation and has nothing on disk worth keeping.
|
||||||
self.transmission.remove_torrent(added.id, true).await?;
|
self.transmission.remove_torrent(added.id, true).await?;
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
movie_id = movie.id,
|
movie_id = target.movie_id,
|
||||||
release = release_name,
|
release = release_name,
|
||||||
infohash = added.hash,
|
infohash = added.hash,
|
||||||
"blacklisted torrent re-listed under a new name; removed"
|
"blacklisted torrent re-listed under a new name; removed"
|
||||||
@@ -652,13 +640,13 @@ struct PendingMovie {
|
|||||||
|
|
||||||
/// The eligible view of a stored release, ranked for selection.
|
/// The eligible view of a stored release, ranked for selection.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct Eligible {
|
pub(crate) struct Eligible {
|
||||||
id: i64,
|
id: i64,
|
||||||
indexer_id: i64,
|
pub(crate) indexer_id: i64,
|
||||||
guid: String,
|
pub(crate) guid: String,
|
||||||
name: String,
|
pub(crate) name: String,
|
||||||
download_url: String,
|
download_url: String,
|
||||||
score: i64,
|
pub(crate) score: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The gap, straight out of the domain rows (§8).
|
/// The gap, straight out of the domain rows (§8).
|
||||||
@@ -759,7 +747,7 @@ fn is_digitally_released(digital_release: Option<&str>) -> bool {
|
|||||||
/// trigger (§6.3: "including by RSS") and so the stored row says why — a
|
/// trigger (§6.3: "including by RSS") and so the stored row says why — a
|
||||||
/// blacklisted release is rejected under the `blacklisted` rule, which is
|
/// blacklisted release is rejected under the `blacklisted` rule, which is
|
||||||
/// what stops §9.3's manual view from offering it as a clean match.
|
/// what stops §9.3's manual view from offering it as a clean match.
|
||||||
async fn store_release(
|
pub(crate) async fn store_release(
|
||||||
database: &Db,
|
database: &Db,
|
||||||
movie_id: i64,
|
movie_id: i64,
|
||||||
release: &SearchRelease,
|
release: &SearchRelease,
|
||||||
@@ -919,6 +907,7 @@ fn rfc3339(time: SystemTime) -> Option<String> {
|
|||||||
#[allow(clippy::unwrap_used)]
|
#[allow(clippy::unwrap_used)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use wiremock::matchers::{method, path, query_param};
|
use wiremock::matchers::{method, path, query_param};
|
||||||
@@ -1697,14 +1686,17 @@ mod tests {
|
|||||||
let (downloader, fake) = transmission().await;
|
let (downloader, fake) = transmission().await;
|
||||||
|
|
||||||
let mut action = action(&indexer, &downloader);
|
let mut action = action(&indexer, &downloader);
|
||||||
action.discovery_timeout = Duration::from_millis(50);
|
action.indexers.discovery_timeout = Duration::from_millis(50);
|
||||||
|
|
||||||
let error = action
|
let error = action
|
||||||
.tick(&database)
|
.tick(&database)
|
||||||
.await
|
.await
|
||||||
.expect_err("discovery cannot complete");
|
.expect_err("discovery cannot complete");
|
||||||
|
|
||||||
assert!(matches!(error, GrabError::IndexerDiscoveryTimeout(_)));
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
GrabError::Discovery(DiscoveryError::Timeout(_))
|
||||||
|
));
|
||||||
assert!(fake.torrents().is_empty());
|
assert!(fake.torrents().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
//! The searchable indexer list, discovered once and reused. See DESIGN.md
|
||||||
|
//! §6.1.
|
||||||
|
//!
|
||||||
|
//! Prowlarr enumerates one indexer per call and probes `t=caps` for each,
|
||||||
|
//! sequentially, so both the 30 s grab lane and the 10 min RSS lane would
|
||||||
|
//! spend most of their budget rediscovering the same list. Each lane keeps
|
||||||
|
//! its own directory: they run concurrently and neither should wait on the
|
||||||
|
//! other's HTTP calls.
|
||||||
|
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use arr_indexer::ProwlarrClient;
|
||||||
|
|
||||||
|
/// How long a discovered indexer list is reused.
|
||||||
|
const CACHE_TTL: Duration = Duration::from_mins(15);
|
||||||
|
|
||||||
|
/// The share of a tick's budget discovery may spend. One unresponsive
|
||||||
|
/// tracker must not cancel the whole action before a single search has run;
|
||||||
|
/// past this the last known list is used instead.
|
||||||
|
const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(8);
|
||||||
|
|
||||||
|
/// A discovery that produced nothing usable at all.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum DiscoveryError {
|
||||||
|
#[error("prowlarr: {0}")]
|
||||||
|
Prowlarr(#[from] arr_indexer::Error),
|
||||||
|
#[error("indexer discovery timed out after {0:?} and none is known yet")]
|
||||||
|
Timeout(Duration),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Cache {
|
||||||
|
ids: Vec<i64>,
|
||||||
|
refreshed_at: Option<Instant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The indexers that accept a text search, cached across ticks.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct IndexerDirectory {
|
||||||
|
prowlarr: ProwlarrClient,
|
||||||
|
cache: tokio::sync::RwLock<Cache>,
|
||||||
|
/// [`DISCOVERY_TIMEOUT`], overridden by tests that cannot wait out the
|
||||||
|
/// real one. Mirrors `ReconcileLoop`'s action timeout override.
|
||||||
|
pub(crate) discovery_timeout: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IndexerDirectory {
|
||||||
|
pub fn new(prowlarr: ProwlarrClient) -> Self {
|
||||||
|
Self {
|
||||||
|
prowlarr,
|
||||||
|
cache: tokio::sync::RwLock::new(Cache::default()),
|
||||||
|
discovery_timeout: DISCOVERY_TIMEOUT,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ids of every indexer advertising a text search — which is also
|
||||||
|
/// the RSS feed, an empty-query text search (§6.2).
|
||||||
|
///
|
||||||
|
/// A refresh that fails or times out keeps the previous list rather than
|
||||||
|
/// stopping the tick, and only an empty cache turns that into an error.
|
||||||
|
pub async fn searchable(&self) -> Result<Vec<i64>, DiscoveryError> {
|
||||||
|
{
|
||||||
|
let cache = self.cache.read().await;
|
||||||
|
if cache
|
||||||
|
.refreshed_at
|
||||||
|
.is_some_and(|at| at.elapsed() < CACHE_TTL)
|
||||||
|
{
|
||||||
|
return Ok(cache.ids.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let discovered =
|
||||||
|
tokio::time::timeout(self.discovery_timeout, self.prowlarr.indexers()).await;
|
||||||
|
let mut cache = self.cache.write().await;
|
||||||
|
match discovered {
|
||||||
|
Ok(Ok(indexers)) => {
|
||||||
|
cache.ids = indexers
|
||||||
|
.iter()
|
||||||
|
.filter(|indexer| indexer.capabilities.search.available)
|
||||||
|
.map(|indexer| indexer.id)
|
||||||
|
.collect();
|
||||||
|
cache.refreshed_at = Some(Instant::now());
|
||||||
|
}
|
||||||
|
Ok(Err(error)) if cache.ids.is_empty() => return Err(error.into()),
|
||||||
|
Err(_) if cache.ids.is_empty() => {
|
||||||
|
return Err(DiscoveryError::Timeout(self.discovery_timeout))
|
||||||
|
}
|
||||||
|
Ok(Err(error)) => {
|
||||||
|
tracing::warn!(%error, "indexer discovery failed; using the last known list");
|
||||||
|
}
|
||||||
|
Err(_) => tracing::warn!(
|
||||||
|
timeout_seconds = self.discovery_timeout.as_secs(),
|
||||||
|
"indexer discovery timed out; using the last known list"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
Ok(cache.ids.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,8 +3,10 @@
|
|||||||
mod config;
|
mod config;
|
||||||
mod grab;
|
mod grab;
|
||||||
mod import;
|
mod import;
|
||||||
|
mod indexers;
|
||||||
mod reaper;
|
mod reaper;
|
||||||
pub mod reconcile;
|
pub mod reconcile;
|
||||||
|
mod rss;
|
||||||
mod web;
|
mod web;
|
||||||
|
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
@@ -19,6 +21,7 @@ use grab::{GrabAction, SeedingLimits, SeedingRules};
|
|||||||
use import::ImportAction;
|
use import::ImportAction;
|
||||||
use reaper::ReaperAction;
|
use reaper::ReaperAction;
|
||||||
use reconcile::{ReconcileLoop, Tick};
|
use reconcile::{ReconcileLoop, Tick};
|
||||||
|
use rss::RssAction;
|
||||||
use tower_http::trace::TraceLayer;
|
use tower_http::trace::TraceLayer;
|
||||||
|
|
||||||
/// Dump the `OpenAPI` document and exit, instead of serving. `just gen-client`
|
/// Dump the `OpenAPI` document and exit, instead of serving. `just gen-client`
|
||||||
@@ -103,49 +106,7 @@ async fn run() -> Result<(), Error> {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let mut reconcile = ReconcileLoop::new(database.clone());
|
let reconcile = reconcile_loop(&database, &config, &transmission, tmdb.as_ref())?;
|
||||||
// Without a Prowlarr key nothing can be searched, so the grab lane stays
|
|
||||||
// unregistered rather than failing a tick every 30 seconds.
|
|
||||||
if let (Some(key), Some(tmdb)) = (config.prowlarr_api_key.clone(), tmdb.clone()) {
|
|
||||||
let prowlarr = arr_indexer::ProwlarrClient::new(config.prowlarr_url.clone(), key)?;
|
|
||||||
reconcile = reconcile.register(
|
|
||||||
Tick::Reconcile,
|
|
||||||
GrabAction::new(
|
|
||||||
prowlarr,
|
|
||||||
transmission.clone(),
|
|
||||||
config.download_dir.clone(),
|
|
||||||
SeedingRules::new(
|
|
||||||
SeedingLimits {
|
|
||||||
ratio: config.seed_ratio_limit,
|
|
||||||
idle_minutes: config.seed_idle_limit_minutes,
|
|
||||||
},
|
|
||||||
config
|
|
||||||
.tracker_seeding
|
|
||||||
.iter()
|
|
||||||
.map(|(&id, rule)| {
|
|
||||||
(
|
|
||||||
id,
|
|
||||||
SeedingLimits {
|
|
||||||
ratio: rule.ratio,
|
|
||||||
idle_minutes: rule.min_seed_time,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.with_tmdb(tmdb),
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
tracing::warn!("Prowlarr or TMDB is not configured: nothing will be grabbed");
|
|
||||||
}
|
|
||||||
// Grab before import, so a download that completes on this tick is
|
|
||||||
// imported on this tick.
|
|
||||||
reconcile = reconcile.register(
|
|
||||||
Tick::Reconcile,
|
|
||||||
ImportAction::new(transmission.clone(), arr_probe::Prober::new()),
|
|
||||||
);
|
|
||||||
reconcile = reconcile.register(Tick::Reaper, ReaperAction::new(transmission));
|
|
||||||
|
|
||||||
// Jellyseerr's Radarr shim (DESIGN.md §9.4) reads the same database and
|
// Jellyseerr's Radarr shim (DESIGN.md §9.4) reads the same database and
|
||||||
// needs its own TMDB client for `movie/lookup`.
|
// needs its own TMDB client for `movie/lookup`.
|
||||||
@@ -202,6 +163,77 @@ async fn run() -> Result<(), Error> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wire the reconcile lanes (DESIGN.md §8). Grab and RSS both need a
|
||||||
|
/// Prowlarr key and grab needs TMDB as well; a lane whose upstream is not
|
||||||
|
/// configured stays unregistered rather than failing every tick.
|
||||||
|
fn reconcile_loop(
|
||||||
|
database: &Db,
|
||||||
|
config: &Config,
|
||||||
|
transmission: &arr_dl::TransmissionClient,
|
||||||
|
tmdb: Option<&Arc<TmdbClient>>,
|
||||||
|
) -> Result<ReconcileLoop, Error> {
|
||||||
|
let mut reconcile = ReconcileLoop::new(database.clone());
|
||||||
|
let seeding = SeedingRules::new(
|
||||||
|
SeedingLimits {
|
||||||
|
ratio: config.seed_ratio_limit,
|
||||||
|
idle_minutes: config.seed_idle_limit_minutes,
|
||||||
|
},
|
||||||
|
config
|
||||||
|
.tracker_seeding
|
||||||
|
.iter()
|
||||||
|
.map(|(&id, rule)| {
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
SeedingLimits {
|
||||||
|
ratio: rule.ratio,
|
||||||
|
idle_minutes: rule.min_seed_time,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
let prowlarr = config
|
||||||
|
.prowlarr_api_key
|
||||||
|
.clone()
|
||||||
|
.map(|key| arr_indexer::ProwlarrClient::new(config.prowlarr_url.clone(), key))
|
||||||
|
.transpose()?;
|
||||||
|
|
||||||
|
if let (Some(prowlarr), Some(tmdb)) = (prowlarr.as_ref(), tmdb) {
|
||||||
|
reconcile = reconcile.register(
|
||||||
|
Tick::Reconcile,
|
||||||
|
GrabAction::new(
|
||||||
|
prowlarr.clone(),
|
||||||
|
transmission.clone(),
|
||||||
|
config.download_dir.clone(),
|
||||||
|
seeding.clone(),
|
||||||
|
)
|
||||||
|
.with_tmdb(Arc::clone(tmdb)),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
tracing::warn!("Prowlarr or TMDB is not configured: nothing will be grabbed");
|
||||||
|
}
|
||||||
|
// RSS needs no TMDB: it matches what the feeds already carry against the
|
||||||
|
// wanted list (§6.2).
|
||||||
|
if let Some(prowlarr) = prowlarr {
|
||||||
|
reconcile = reconcile.register(
|
||||||
|
Tick::Rss,
|
||||||
|
RssAction::new(
|
||||||
|
prowlarr,
|
||||||
|
transmission.clone(),
|
||||||
|
config.download_dir.clone(),
|
||||||
|
seeding,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Grab before import, so a download that completes on this tick is
|
||||||
|
// imported on this tick.
|
||||||
|
reconcile = reconcile.register(
|
||||||
|
Tick::Reconcile,
|
||||||
|
ImportAction::new(transmission.clone(), arr_probe::Prober::new()),
|
||||||
|
);
|
||||||
|
Ok(reconcile.register(Tick::Reaper, ReaperAction::new(transmission.clone())))
|
||||||
|
}
|
||||||
|
|
||||||
/// Stop accepting on Ctrl-C, or on the SIGTERM a service manager sends.
|
/// Stop accepting on Ctrl-C, or on the SIGTERM a service manager sends.
|
||||||
async fn shutdown() {
|
async fn shutdown() {
|
||||||
let interrupt = async {
|
let interrupt = async {
|
||||||
|
|||||||
@@ -0,0 +1,574 @@
|
|||||||
|
//! RSS sync: one empty-query Torznab call per indexer, matched against the
|
||||||
|
//! whole wanted list locally. See DESIGN.md §6.2.
|
||||||
|
//!
|
||||||
|
//! The property this lane is built around is that its cost is one call per
|
||||||
|
//! indexer, whatever the library holds. Nothing here is per-title, so every
|
||||||
|
//! wanted item is compared against every result on every pass, forever:
|
||||||
|
//! there is no backoff, no attempt counter, and no release-date gate. Those
|
||||||
|
//! belong to targeted search, which pays per title.
|
||||||
|
//!
|
||||||
|
//! `blocked` is honoured the other way round from targeted search (§6.3): it
|
||||||
|
//! stops a title being searched for, and leaves it matching RSS.
|
||||||
|
|
||||||
|
use std::collections::hash_map::Entry;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use arr_core::matching::{match_movie, MatchKind, WantedMovie};
|
||||||
|
use arr_core::{Language, MovieId};
|
||||||
|
use arr_db::{Blacklist, Db, MoviePolicy};
|
||||||
|
use arr_dl::TransmissionClient;
|
||||||
|
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
|
||||||
|
|
||||||
|
use crate::grab::{store_release, Eligible, GrabError, GrabTarget, Grabber, SeedingRules};
|
||||||
|
use crate::indexers::IndexerDirectory;
|
||||||
|
use crate::reconcile::{Action, ActionFuture, Outcome};
|
||||||
|
|
||||||
|
/// Reads every indexer's feed and grabs what it can tie to a wanted title.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct RssAction {
|
||||||
|
prowlarr: ProwlarrClient,
|
||||||
|
grabber: Grabber,
|
||||||
|
indexers: IndexerDirectory,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RssAction {
|
||||||
|
#[must_use]
|
||||||
|
pub fn new(
|
||||||
|
prowlarr: ProwlarrClient,
|
||||||
|
transmission: TransmissionClient,
|
||||||
|
download_dir: PathBuf,
|
||||||
|
seeding: SeedingRules,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
indexers: IndexerDirectory::new(prowlarr.clone()),
|
||||||
|
prowlarr,
|
||||||
|
grabber: Grabber::new(transmission, download_dir, seeding),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn tick(&self, database: &Db) -> Result<Vec<Outcome>, GrabError> {
|
||||||
|
let wanted = wanted_movies(database).await?;
|
||||||
|
if wanted.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
let indexers = self.indexers.searchable().await?;
|
||||||
|
if indexers.is_empty() {
|
||||||
|
tracing::warn!("no indexer advertises a text search; no RSS feed to read");
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let releases = self.feeds(&indexers).await;
|
||||||
|
let blacklist = Blacklist::load(database.pool()).await?;
|
||||||
|
let winners = self
|
||||||
|
.match_feeds(database, &wanted, releases, &blacklist)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut outcomes = Vec::new();
|
||||||
|
for (movie, winner) in winners {
|
||||||
|
let outcome = self
|
||||||
|
.grabber
|
||||||
|
.send_winner(
|
||||||
|
database,
|
||||||
|
&GrabTarget {
|
||||||
|
movie_id: movie.id,
|
||||||
|
title: &movie.title,
|
||||||
|
// §6.2: RSS never backs off, so a failed grab here
|
||||||
|
// must not spend one of targeted search's attempts.
|
||||||
|
counts_as_attempt: false,
|
||||||
|
},
|
||||||
|
&movie.policy,
|
||||||
|
&blacklist,
|
||||||
|
winner,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
outcomes.extend(outcome);
|
||||||
|
}
|
||||||
|
Ok(outcomes)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One empty-query call per indexer (§6.2). A tracker that fails is
|
||||||
|
/// logged and skipped: the next pass is ten minutes away.
|
||||||
|
async fn feeds(&self, indexers: &[i64]) -> Vec<SearchRelease> {
|
||||||
|
let mut releases = Vec::new();
|
||||||
|
for search in self
|
||||||
|
.prowlarr
|
||||||
|
.search_indexers(indexers, &SearchRequest::Rss)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
if let Some(error) = search.error {
|
||||||
|
tracing::warn!(indexer_id = search.indexer_id, %error, "RSS feed failed");
|
||||||
|
}
|
||||||
|
releases.extend(search.releases);
|
||||||
|
}
|
||||||
|
releases
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tie each feed item to a wanted title, cache it with its verdict, and
|
||||||
|
/// keep the best eligible candidate per title.
|
||||||
|
///
|
||||||
|
/// One title takes at most one torrent per pass, and the ordering is the
|
||||||
|
/// same total order targeted search uses, so the two triggers agree on
|
||||||
|
/// which release wins.
|
||||||
|
async fn match_feeds(
|
||||||
|
&self,
|
||||||
|
database: &Db,
|
||||||
|
wanted: &[WantedMovie],
|
||||||
|
releases: Vec<SearchRelease>,
|
||||||
|
blacklist: &Blacklist,
|
||||||
|
) -> Result<Vec<(Grabbable, Eligible)>, GrabError> {
|
||||||
|
let mut policies: HashMap<i64, Option<Grabbable>> = HashMap::new();
|
||||||
|
let mut best: HashMap<i64, Eligible> = HashMap::new();
|
||||||
|
|
||||||
|
for release in releases {
|
||||||
|
let claims = arr_parse::parse(&release.name);
|
||||||
|
let Some(matched) = match_movie(wanted, release.tmdb_id, &claims) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let movie_id = matched.movie.0;
|
||||||
|
let grabbable = match policies.entry(movie_id) {
|
||||||
|
Entry::Occupied(entry) => entry.into_mut(),
|
||||||
|
Entry::Vacant(entry) => {
|
||||||
|
entry.insert(load_grabbable(database, wanted, movie_id).await?)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let Some(grabbable) = grabbable.as_ref() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
tracing::info!(
|
||||||
|
movie_id,
|
||||||
|
release = release.name,
|
||||||
|
by = match matched.kind {
|
||||||
|
MatchKind::TmdbId => "tmdb id",
|
||||||
|
MatchKind::TitleAndYear => "title and year",
|
||||||
|
},
|
||||||
|
"RSS result matched a wanted title"
|
||||||
|
);
|
||||||
|
let stored = store_release(
|
||||||
|
database,
|
||||||
|
movie_id,
|
||||||
|
&release,
|
||||||
|
&grabbable.policy.policy,
|
||||||
|
&grabbable.policy.overrides,
|
||||||
|
&grabbable.language,
|
||||||
|
blacklist,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let Some(candidate) = stored else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let incumbent = best.get(&movie_id);
|
||||||
|
if incumbent.is_none_or(|incumbent| beats(&candidate, incumbent)) {
|
||||||
|
best.insert(movie_id, candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut winners: Vec<(Grabbable, Eligible)> = best
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(movie_id, candidate)| {
|
||||||
|
let grabbable = policies.get(&movie_id).and_then(Clone::clone)?;
|
||||||
|
Some((grabbable, candidate))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
winners.sort_by_key(|(movie, _)| movie.id);
|
||||||
|
Ok(winners)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Action for RssAction {
|
||||||
|
fn name(&self) -> &'static str {
|
||||||
|
"rss"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run<'a>(&'a self, database: &'a Db) -> ActionFuture<'a> {
|
||||||
|
Box::pin(async move { self.tick(database).await.map_err(Into::into) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A matched title with everything a grab needs loaded.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct Grabbable {
|
||||||
|
id: i64,
|
||||||
|
title: String,
|
||||||
|
policy: MoviePolicy,
|
||||||
|
language: Language,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Better score first, then the tiebreaks targeted search sorts by, so both
|
||||||
|
/// triggers pick the same release out of the same candidates.
|
||||||
|
fn beats(candidate: &Eligible, incumbent: &Eligible) -> bool {
|
||||||
|
candidate
|
||||||
|
.score
|
||||||
|
.cmp(&incumbent.score)
|
||||||
|
.then_with(|| incumbent.indexer_id.cmp(&candidate.indexer_id))
|
||||||
|
.then_with(|| incumbent.guid.cmp(&candidate.guid))
|
||||||
|
== std::cmp::Ordering::Greater
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every wanted title with neither a file nor a grab in flight.
|
||||||
|
///
|
||||||
|
/// Unlike targeted search's work list this is not limited or ordered by when
|
||||||
|
/// a title was last searched: matching is local, so the cost of carrying the
|
||||||
|
/// whole list is a string comparison per title per feed item (§6.2). Blocked
|
||||||
|
/// titles are here on purpose (§6.3).
|
||||||
|
async fn wanted_movies(database: &Db) -> Result<Vec<WantedMovie>, GrabError> {
|
||||||
|
let rows = sqlx::query!(
|
||||||
|
r#"
|
||||||
|
SELECT m.id AS "id!: i64",
|
||||||
|
m.tmdb_id AS "tmdb_id!: i64",
|
||||||
|
m.title AS "title!: String",
|
||||||
|
m.year
|
||||||
|
FROM movies m
|
||||||
|
WHERE m.wanted = 1
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM media_files f
|
||||||
|
WHERE f.owner_kind = 'movie' AND f.owner_id = m.id
|
||||||
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM grabs g
|
||||||
|
WHERE g.target_kind = 'movie' AND g.target_id = m.id
|
||||||
|
AND g.state IN ('sent', 'downloaded', 'imported')
|
||||||
|
)
|
||||||
|
ORDER BY m.id
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.fetch_all(database.pool())
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|row| WantedMovie {
|
||||||
|
id: MovieId(row.id),
|
||||||
|
tmdb_id: u32::try_from(row.tmdb_id).ok(),
|
||||||
|
title: row.title,
|
||||||
|
year: row.year.and_then(|year| u16::try_from(year).ok()),
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The policy and original language a matched title needs, or `None` when
|
||||||
|
/// the title cannot be evaluated yet.
|
||||||
|
async fn load_grabbable(
|
||||||
|
database: &Db,
|
||||||
|
wanted: &[WantedMovie],
|
||||||
|
movie_id: i64,
|
||||||
|
) -> Result<Option<Grabbable>, GrabError> {
|
||||||
|
let Some(title) = wanted
|
||||||
|
.iter()
|
||||||
|
.find(|movie| movie.id == MovieId(movie_id))
|
||||||
|
.map(|movie| movie.title.clone())
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let Some(policy) = database.movie_policy(movie_id).await? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
// §5.2: every language rule is expressed against the title's own
|
||||||
|
// original language, and guessing it is worse than not grabbing.
|
||||||
|
let language: Option<String> = sqlx::query_scalar!(
|
||||||
|
"SELECT original_language FROM movies WHERE id = ?",
|
||||||
|
movie_id
|
||||||
|
)
|
||||||
|
.fetch_optional(database.pool())
|
||||||
|
.await?
|
||||||
|
.flatten();
|
||||||
|
let Some(language) = language else {
|
||||||
|
tracing::warn!(movie_id, "no original language yet; not grabbing from RSS");
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(Some(Grabbable {
|
||||||
|
id: movie_id,
|
||||||
|
title,
|
||||||
|
policy,
|
||||||
|
language: arr_db::policy::language(&language),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use wiremock::matchers::{method, path, query_param, query_param_is_missing};
|
||||||
|
use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::grab::SeedingLimits;
|
||||||
|
use arr_dl::TransmissionClient;
|
||||||
|
|
||||||
|
/// The recorded feed: two releases of one wanted title, one that only a
|
||||||
|
/// TMDB id identifies, one near miss, one TV item and one film nobody
|
||||||
|
/// asked for.
|
||||||
|
const FEED: &str = include_str!("../tests/fixtures/rss.xml");
|
||||||
|
|
||||||
|
const INDEXERS: [i64; 2] = [7, 9];
|
||||||
|
|
||||||
|
/// Enough of Transmission to add a torrent and list nothing back.
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
struct FakeTransmission {
|
||||||
|
added: Arc<Mutex<Vec<String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Respond for FakeTransmission {
|
||||||
|
fn respond(&self, request: &Request) -> ResponseTemplate {
|
||||||
|
let body: Value = serde_json::from_slice(&request.body).unwrap();
|
||||||
|
match body["method"].as_str().unwrap_or_default() {
|
||||||
|
"torrent-add" => {
|
||||||
|
let source = body["arguments"]["filename"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_owned();
|
||||||
|
let mut added = self.added.lock().unwrap();
|
||||||
|
added.push(source.clone());
|
||||||
|
let id = i64::try_from(added.len()).unwrap();
|
||||||
|
success(&json!({"torrent-added": {
|
||||||
|
"id": id, "name": source, "hashString": format!("{id:040x}")
|
||||||
|
}}))
|
||||||
|
}
|
||||||
|
"torrent-get" => success(&json!({"torrents": []})),
|
||||||
|
_ => success(&json!({})),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn success(arguments: &Value) -> ResponseTemplate {
|
||||||
|
ResponseTemplate::new(200).set_body_json(json!({
|
||||||
|
"result": "success", "arguments": arguments
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Two indexers, both advertising a text search, both serving the same
|
||||||
|
/// feed to an empty query.
|
||||||
|
async fn prowlarr() -> MockServer {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/indexer"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(json!([
|
||||||
|
{"id": 7, "name": "alpha", "enable": true},
|
||||||
|
{"id": 9, "name": "beta", "enable": true},
|
||||||
|
])))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
for id in INDEXERS {
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path(format!("/{id}/api")))
|
||||||
|
.and(query_param("t", "caps"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(
|
||||||
|
r#"<caps><searching><search available="yes" supportedParams="q"/></searching></caps>"#,
|
||||||
|
))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
// The RSS call is `t=search` with no `q` at all (§6.2).
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path(format!("/{id}/api")))
|
||||||
|
.and(query_param("t", "search"))
|
||||||
|
.and(query_param_is_missing("q"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(FEED))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
server
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn transmission() -> (MockServer, FakeTransmission) {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let fake = FakeTransmission::default();
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.respond_with(fake.clone())
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
(server, fake)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `(tmdb_id, title, year, blocked)`.
|
||||||
|
async fn wanted(movies: &[(i64, &str, i64, bool)]) -> (tempfile::TempDir, Db) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let database = Db::connect(dir.path().join("arr.db")).await.unwrap();
|
||||||
|
database.migrate().await.unwrap();
|
||||||
|
for &(tmdb_id, title, year, blocked) in movies {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO movies (tmdb_id, title, year, original_language, blocked, root_id)
|
||||||
|
SELECT ?, ?, ?, 'en', ?, id
|
||||||
|
FROM roots WHERE kind = 'movie' AND audience = 'main'",
|
||||||
|
)
|
||||||
|
.bind(tmdb_id)
|
||||||
|
.bind(title)
|
||||||
|
.bind(year)
|
||||||
|
.bind(i64::from(blocked))
|
||||||
|
.execute(database.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
(dir, database)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn action(prowlarr: &MockServer, transmission: &MockServer) -> RssAction {
|
||||||
|
RssAction::new(
|
||||||
|
ProwlarrClient::new(prowlarr.uri(), "key").unwrap(),
|
||||||
|
TransmissionClient::new(&transmission.uri()).unwrap(),
|
||||||
|
PathBuf::from("/mnt/media/transmission/complete"),
|
||||||
|
SeedingRules::new(
|
||||||
|
SeedingLimits {
|
||||||
|
ratio: 1.0,
|
||||||
|
idle_minutes: 60,
|
||||||
|
},
|
||||||
|
HashMap::new(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `(movie title, grabbed release name)` for every grab that was made.
|
||||||
|
async fn grabbed(database: &Db) -> Vec<(String, String)> {
|
||||||
|
sqlx::query_as::<_, (String, String)>(
|
||||||
|
"SELECT m.title, r.name
|
||||||
|
FROM grabs g
|
||||||
|
JOIN movies m ON m.id = g.target_id
|
||||||
|
JOIN releases r ON r.id = g.release_id
|
||||||
|
ORDER BY m.title",
|
||||||
|
)
|
||||||
|
.fetch_all(database.pool())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stored_releases(database: &Db) -> Vec<String> {
|
||||||
|
sqlx::query_scalar::<_, String>("SELECT DISTINCT guid FROM releases ORDER BY guid")
|
||||||
|
.fetch_all(database.pool())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The acceptance case: a recorded feed against a seeded wanted list
|
||||||
|
/// grabs exactly the intended titles, by TMDB id where the tracker
|
||||||
|
/// supplied one.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exactly_the_intended_titles_match() {
|
||||||
|
let (_dir, database) = wanted(&[
|
||||||
|
(693_134, "Dune: Part Two", 2024, false),
|
||||||
|
(933_260, "The Substance", 2024, false),
|
||||||
|
(438_631, "Dune", 2021, false),
|
||||||
|
])
|
||||||
|
.await;
|
||||||
|
let indexer = prowlarr().await;
|
||||||
|
let (downloader, fake) = transmission().await;
|
||||||
|
|
||||||
|
let outcomes = action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(outcomes.len(), 3);
|
||||||
|
assert_eq!(
|
||||||
|
grabbed(&database).await,
|
||||||
|
vec![
|
||||||
|
(
|
||||||
|
"Dune".to_owned(),
|
||||||
|
"Dune.2021.2160p.WEB-DL.DDP5.1.Atmos.H.265-GROUP".to_owned()
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Dune: Part Two".to_owned(),
|
||||||
|
"Dune.Part.Two.2024.2160p.WEB-DL.DDP5.1.Atmos.H.265-GROUP".to_owned()
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"The Substance".to_owned(),
|
||||||
|
"La.Sustancia.2024.2160p.WEB-DL.DDP5.1.H.265-GRUPO".to_owned()
|
||||||
|
),
|
||||||
|
],
|
||||||
|
"one torrent per title, and the 1080p Dune: Part Two loses to the 2160p one"
|
||||||
|
);
|
||||||
|
assert_eq!(fake.added.lock().unwrap().len(), 3);
|
||||||
|
// The TV item and the film nobody asked for are not cached against
|
||||||
|
// any title: an unmatched release belongs to nothing. Both indexers
|
||||||
|
// carry the same four, under their own indexer id.
|
||||||
|
assert_eq!(
|
||||||
|
stored_releases(&database).await,
|
||||||
|
vec!["alpha-1", "alpha-2", "alpha-3", "alpha-4"]
|
||||||
|
);
|
||||||
|
let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM releases")
|
||||||
|
.fetch_one(database.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rows, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The near miss. "Dune: Part Three" is one word from a release that is
|
||||||
|
/// in the feed, from the same franchise, in the same year.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_near_miss_is_not_grabbed() {
|
||||||
|
let (_dir, database) = wanted(&[(9_999_999, "Dune: Part Three", 2024, false)]).await;
|
||||||
|
let indexer = prowlarr().await;
|
||||||
|
let (downloader, fake) = transmission().await;
|
||||||
|
|
||||||
|
let outcomes = action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
assert!(outcomes.is_empty());
|
||||||
|
assert!(fake.added.lock().unwrap().is_empty());
|
||||||
|
assert!(stored_releases(&database).await.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// §6.2: RSS never backs off, so nothing it does may spend one of
|
||||||
|
/// targeted search's attempts.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn matching_never_touches_the_search_backoff() {
|
||||||
|
let (_dir, database) = wanted(&[
|
||||||
|
(693_134, "Dune: Part Two", 2024, false),
|
||||||
|
(9_999_999, "Dune: Part Three", 2024, false),
|
||||||
|
])
|
||||||
|
.await;
|
||||||
|
let indexer = prowlarr().await;
|
||||||
|
let (downloader, _fake) = transmission().await;
|
||||||
|
|
||||||
|
action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
let attempts: Vec<(i64, Option<String>)> =
|
||||||
|
sqlx::query_as("SELECT search_attempts, last_searched_at FROM movies ORDER BY id")
|
||||||
|
.fetch_all(database.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(attempts, vec![(0, None), (0, None)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// §6.3: `blocked` stops targeted search and leaves RSS matching on.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_blocked_title_still_matches_rss() {
|
||||||
|
let (_dir, database) = wanted(&[(693_134, "Dune: Part Two", 2024, true)]).await;
|
||||||
|
let indexer = prowlarr().await;
|
||||||
|
let (downloader, _fake) = transmission().await;
|
||||||
|
|
||||||
|
let outcomes = action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(outcomes.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The property the whole lane exists for: cost is one call per indexer,
|
||||||
|
/// whatever the wanted list holds.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn one_empty_query_call_per_indexer() {
|
||||||
|
let (_dir, database) = wanted(&[
|
||||||
|
(693_134, "Dune: Part Two", 2024, false),
|
||||||
|
(933_260, "The Substance", 2024, false),
|
||||||
|
(438_631, "Dune", 2021, false),
|
||||||
|
])
|
||||||
|
.await;
|
||||||
|
let indexer = prowlarr().await;
|
||||||
|
let (downloader, _fake) = transmission().await;
|
||||||
|
|
||||||
|
action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
for id in INDEXERS {
|
||||||
|
let feeds = indexer
|
||||||
|
.received_requests()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|request| {
|
||||||
|
request.url.path() == format!("/{id}/api")
|
||||||
|
&& request
|
||||||
|
.url
|
||||||
|
.query_pairs()
|
||||||
|
.any(|(name, value)| name == "t" && value == "search")
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
assert_eq!(feeds, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+61
@@ -0,0 +1,61 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<rss version="2.0" xmlns:torznab="http://torznab.com/schemas/2015/feed">
|
||||||
|
<channel>
|
||||||
|
<title>Alpha Tracker</title>
|
||||||
|
<item>
|
||||||
|
<title>Dune.Part.Two.2024.2160p.WEB-DL.DDP5.1.Atmos.H.265-GROUP</title>
|
||||||
|
<guid>alpha-1</guid>
|
||||||
|
<link>https://indexer.invalid/download/alpha-1</link>
|
||||||
|
<pubDate>Sat, 22 Aug 2026 09:14:00 +0000</pubDate>
|
||||||
|
<enclosure url="https://indexer.invalid/download/alpha-1" length="23622320128"
|
||||||
|
type="application/x-bittorrent"/>
|
||||||
|
<torznab:attr name="seeders" value="41"/>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Dune.Part.Two.2024.1080p.WEB-DL.DD5.1.H.264-OTHER</title>
|
||||||
|
<guid>alpha-2</guid>
|
||||||
|
<link>https://indexer.invalid/download/alpha-2</link>
|
||||||
|
<pubDate>Sat, 22 Aug 2026 08:02:00 +0000</pubDate>
|
||||||
|
<enclosure url="https://indexer.invalid/download/alpha-2" length="8589934592"
|
||||||
|
type="application/x-bittorrent"/>
|
||||||
|
<torznab:attr name="seeders" value="12"/>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>La.Sustancia.2024.2160p.WEB-DL.DDP5.1.H.265-GRUPO</title>
|
||||||
|
<guid>alpha-3</guid>
|
||||||
|
<link>https://indexer.invalid/download/alpha-3</link>
|
||||||
|
<pubDate>Sat, 22 Aug 2026 07:40:00 +0000</pubDate>
|
||||||
|
<enclosure url="https://indexer.invalid/download/alpha-3" length="21474836480"
|
||||||
|
type="application/x-bittorrent"/>
|
||||||
|
<torznab:attr name="seeders" value="30"/>
|
||||||
|
<torznab:attr name="tmdbid" value="933260"/>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Dune.2021.2160p.WEB-DL.DDP5.1.Atmos.H.265-GROUP</title>
|
||||||
|
<guid>alpha-4</guid>
|
||||||
|
<link>https://indexer.invalid/download/alpha-4</link>
|
||||||
|
<pubDate>Sat, 22 Aug 2026 06:55:00 +0000</pubDate>
|
||||||
|
<enclosure url="https://indexer.invalid/download/alpha-4" length="22548578304"
|
||||||
|
type="application/x-bittorrent"/>
|
||||||
|
<torznab:attr name="seeders" value="88"/>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Dune.Prophecy.S01E03.2024.2160p.WEB-DL.DDP5.1.H.265-GROUP</title>
|
||||||
|
<guid>alpha-5</guid>
|
||||||
|
<link>https://indexer.invalid/download/alpha-5</link>
|
||||||
|
<pubDate>Sat, 22 Aug 2026 06:31:00 +0000</pubDate>
|
||||||
|
<enclosure url="https://indexer.invalid/download/alpha-5" length="6442450944"
|
||||||
|
type="application/x-bittorrent"/>
|
||||||
|
<torznab:attr name="seeders" value="55"/>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Nosferatu.2024.2160p.WEB-DL.DDP5.1.Atmos.H.265-GROUP</title>
|
||||||
|
<guid>alpha-6</guid>
|
||||||
|
<link>https://indexer.invalid/download/alpha-6</link>
|
||||||
|
<pubDate>Sat, 22 Aug 2026 05:12:00 +0000</pubDate>
|
||||||
|
<enclosure url="https://indexer.invalid/download/alpha-6" length="20401094656"
|
||||||
|
type="application/x-bittorrent"/>
|
||||||
|
<torznab:attr name="seeders" value="14"/>
|
||||||
|
</item>
|
||||||
|
</channel>
|
||||||
|
</rss>
|
||||||
@@ -69,6 +69,9 @@ pub struct SearchRelease {
|
|||||||
pub seeders: Option<u32>,
|
pub seeders: Option<u32>,
|
||||||
pub publish_date: Option<SystemTime>,
|
pub publish_date: Option<SystemTime>,
|
||||||
pub download_url: String,
|
pub download_url: String,
|
||||||
|
/// TMDB id the tracker attached to the item, when it attached one. An ID
|
||||||
|
/// the indexer supplies beats anything read off the release name.
|
||||||
|
pub tmdb_id: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Results from one indexer in a multi-indexer search.
|
/// Results from one indexer in a multi-indexer search.
|
||||||
@@ -279,6 +282,7 @@ struct ReleaseBuilder {
|
|||||||
publish_date: Option<SystemTime>,
|
publish_date: Option<SystemTime>,
|
||||||
link: Option<String>,
|
link: Option<String>,
|
||||||
enclosure_url: Option<String>,
|
enclosure_url: Option<String>,
|
||||||
|
tmdb_id: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ReleaseBuilder {
|
impl ReleaseBuilder {
|
||||||
@@ -295,6 +299,7 @@ impl ReleaseBuilder {
|
|||||||
seeders: self.seeders,
|
seeders: self.seeders,
|
||||||
publish_date: self.publish_date,
|
publish_date: self.publish_date,
|
||||||
download_url,
|
download_url,
|
||||||
|
tmdb_id: self.tmdb_id,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -417,6 +422,12 @@ fn read_element_attributes(
|
|||||||
.and_then(|value| value.parse().ok())
|
.and_then(|value| value.parse().ok())
|
||||||
.or(builder.seeders);
|
.or(builder.seeders);
|
||||||
}
|
}
|
||||||
|
Some("tmdbid") => {
|
||||||
|
builder.tmdb_id = value
|
||||||
|
.and_then(|value| value.trim().parse().ok())
|
||||||
|
.filter(|&id| id != 0)
|
||||||
|
.or(builder.tmdb_id);
|
||||||
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -717,6 +728,38 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An id the tracker attached to the item is what RSS matching prefers
|
||||||
|
/// over anything read off the release name (§6.2).
|
||||||
|
#[test]
|
||||||
|
fn a_torznab_tmdb_id_is_kept() {
|
||||||
|
let body = br#"
|
||||||
|
<rss version="2.0" xmlns:torznab="http://torznab.com/schemas/2015/feed">
|
||||||
|
<channel>
|
||||||
|
<item>
|
||||||
|
<title>La.Sustancia.2024.2160p.WEB-DL</title>
|
||||||
|
<guid>identified</guid>
|
||||||
|
<link>https://indexer.invalid/download/identified</link>
|
||||||
|
<torznab:attr name="tmdbid" value="933260"/>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Nosferatu.2024.2160p.WEB-DL</title>
|
||||||
|
<guid>anonymous</guid>
|
||||||
|
<link>https://indexer.invalid/download/anonymous</link>
|
||||||
|
<torznab:attr name="tmdbid" value="0"/>
|
||||||
|
</item>
|
||||||
|
</channel>
|
||||||
|
</rss>
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let releases = parse_releases(3, body).expect("feed structure is valid");
|
||||||
|
|
||||||
|
assert_eq!(releases[0].tmdb_id, Some(933_260));
|
||||||
|
assert_eq!(
|
||||||
|
releases[1].tmdb_id, None,
|
||||||
|
"a zero is the tracker saying it does not know"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// A pack and one episode of that pack carry the same series title, so
|
/// A pack and one episode of that pack carry the same series title, so
|
||||||
/// the tag on the release name is the only thing separating them.
|
/// the tag on the release name is the only thing separating them.
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in New Issue
Block a user