yolo/arr
3
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: match RSS results by IMDb id (#99)
ci / web (push) Successful in 28s
Details
e2e / e2e (push) Successful in 53s
Details
ci / rust (push) Successful in 2m15s
Details

Browse Source
This commit was merged in pull request #99.
This commit is contained in:
naps62-yolo (agent)
2026-08-23 02:44:42 +01:00
parent 3082568171
commit 02d99ae558
11 changed files with 312 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.sqlx/query-369bc5ff5cf6b84b51bfb0b254a4a78db9b8c277a7e276b2ffb04c8064ce7cf8.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "SQLite",
"query": "UPDATE movies\n SET title = ?, year = ?, original_language = ?, digital_release = ?,\n imdb_id = ?,\n metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'),\n search_attempts = 0, last_searched_at = NULL,\n updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')\n WHERE id = ? AND (\n title IS NOT ? OR year IS NOT ? OR original_language IS NOT ?\n OR digital_release IS NOT ? OR imdb_id IS NOT ?\n )",
"describe": {
"columns": [],
"parameters": {
"Right": 11
},
"nullable": []
},
"hash": "369bc5ff5cf6b84b51bfb0b254a4a78db9b8c277a7e276b2ffb04c8064ce7cf8"
}
.sqlx/query-9da028029e93a01bd4be5a3b065556e60ba014a782d297aa01ba4b80dc94b485.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "UPDATE movies\n SET title = ?, year = ?, original_language = ?, digital_release = ?,\n metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'),\n search_attempts = 0, last_searched_at = NULL,\n updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')\n WHERE id = ? AND (\n title IS NOT ? OR year IS NOT ? OR original_language IS NOT ?\n OR digital_release IS NOT ?\n )",
"describe": {
"columns": [],
"parameters": {
"Right": 9
},
"nullable": []
},
"hash": "9da028029e93a01bd4be5a3b065556e60ba014a782d297aa01ba4b80dc94b485"
}
.sqlx/query-d318af0005db414990095d0d86e312f26fd0e57018ba6a4667d53b590080d4a7.json
Generated
-38
View File
@@ -1,38 +0,0 @@
{
"db_name": "SQLite",
"query": "\n SELECT m.id AS \"id!: i64\",\n m.tmdb_id AS \"tmdb_id!: i64\",\n m.title AS \"title!: String\",\n m.year\n FROM movies m\n WHERE m.wanted = 1\n AND NOT EXISTS (\n SELECT 1 FROM media_files f\n WHERE f.owner_kind = 'movie' AND f.owner_id = m.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM grabs g\n WHERE g.target_kind = 'movie' AND g.target_id = m.id\n AND g.state IN ('sent', 'downloaded', 'imported')\n )\n ORDER BY m.id\n ",
"describe": {
"columns": [
{
"name": "id!: i64",
"ordinal": 0,
"type_info": "Integer"
},
{
"name": "tmdb_id!: i64",
"ordinal": 1,
"type_info": "Integer"
},
{
"name": "title!: String",
"ordinal": 2,
"type_info": "Text"
},
{
"name": "year",
"ordinal": 3,
"type_info": "Integer"
}
],
"parameters": {
"Right": 0
},
"nullable": [
false,
false,
false,
true
]
},
"hash": "d318af0005db414990095d0d86e312f26fd0e57018ba6a4667d53b590080d4a7"
}
.sqlx/query-d58ad94e15e4b4db9dde68f418f586bfa8b6ac2eb2cdb556071f6382283d81ae.json
Generated
+44
View File
@@ -0,0 +1,44 @@
{
"db_name": "SQLite",
"query": "\n SELECT m.id AS \"id!: i64\",\n m.tmdb_id AS \"tmdb_id!: i64\",\n m.imdb_id AS \"imdb_id: String\",\n m.title AS \"title!: String\",\n m.year\n FROM movies m\n WHERE m.wanted = 1\n AND NOT EXISTS (\n SELECT 1 FROM media_files f\n WHERE f.owner_kind = 'movie' AND f.owner_id = m.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM grabs g\n WHERE g.target_kind = 'movie' AND g.target_id = m.id\n AND g.state IN ('sent', 'downloaded', 'imported')\n )\n ORDER BY m.id\n ",
"describe": {
"columns": [
{
"name": "id!: i64",
"ordinal": 0,
"type_info": "Integer"
},
{
"name": "tmdb_id!: i64",
"ordinal": 1,
"type_info": "Integer"
},
{
"name": "imdb_id: String",
"ordinal": 2,
"type_info": "Text"
},
{
"name": "title!: String",
"ordinal": 3,
"type_info": "Text"
},
{
"name": "year",
"ordinal": 4,
"type_info": "Integer"
}
],
"parameters": {
"Right": 0
},
"nullable": [
false,
false,
true,
false,
true
]
},
"hash": "d58ad94e15e4b4db9dde68f418f586bfa8b6ac2eb2cdb556071f6382283d81ae"
}
crates/arr-api/src/search.rs
+1
View File
@@ -832,6 +832,7 @@ mod tests {
publish_date: None,
download_url: "https://tracker/release".into(),
tmdb_id: None,
imdb_id: None,
};
let parsed = arr_parse::parse(&release.name);
let core_score = score(&policy, Candidate::PreGrab(&parsed), 0, 8);
crates/arr-core/src/lib.rs
+1 -1
View File
@@ -15,7 +15,7 @@ pub mod tracking;
pub use arr_parse::NameClaims as ParsedRelease;
pub use grabbing::{season_grab_mode, SeasonGrabFacts, SeasonGrabMode};
pub use matching::{match_movie, MatchKind, MovieMatch, WantedMovie};
pub use matching::{match_movie, MatchKind, MovieMatch, ReleaseIds, WantedMovie};
pub use score::{Score, ScoreWeights};
pub use status::{derive_series_status, SeriesStatus};
crates/arr-core/src/matching.rs
+180 -15
View File
@@ -9,6 +9,8 @@
//! - an ID the tracker itself supplied beats anything read off the name, and
//! an ID that matches nothing wanted ends the comparison rather than
//! falling back to the title;
//! - two supplied IDs that disagree about which title this is are an
//! ambiguity like any other, and match nothing;
//! - a title match needs the years to agree, and a wanted title with a known
//! year is never matched by a release that does not state one;
//! - anything that matches two wanted titles matches neither.
@@ -22,6 +24,10 @@ pub struct WantedMovie {
pub id: MovieId,
/// TMDB id, compared against the one the tracker attached to the item.
pub tmdb_id: Option<u32>,
/// `IMDb` id as TMDB spells it (`tt0317248`), compared against the one
/// the tracker attached to the item. Absent until the metadata refresh
/// has run, and for the titles TMDB knows no `IMDb` id for.
pub imdb_id: Option<String>,
pub title: String,
pub year: Option<u16>,
}
@@ -31,10 +37,22 @@ pub struct WantedMovie {
pub enum MatchKind {
/// The tracker supplied a TMDB id and it is one of ours.
TmdbId,
/// The tracker supplied an `IMDb` id and it is one of ours.
ImdbId,
/// The parsed release name and the wanted title agree, years included.
TitleAndYear,
}
/// The ids a tracker attached to a feed item, as it spelled them.
///
/// Both are optional and independent: a definition emits whichever its
/// tracker carries, and `imdbid` is the one most of them carry (§6.2).
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct ReleaseIds {
pub tmdb_id: Option<u32>,
pub imdb_id: Option<String>,
}
/// One wanted title an RSS item was matched to.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MovieMatch {
@@ -45,24 +63,32 @@ pub struct MovieMatch {
/// The wanted title an RSS item belongs to, or `None` when nothing matches
/// conservatively.
///
/// `tmdb_id` is what the indexer attached to the item, not anything inferred
/// `ids` are what the indexer attached to the item, not anything inferred
/// from the name; `claims` is the parsed release name.
#[must_use]
pub fn match_movie(
wanted: &[WantedMovie],
tmdb_id: Option<u32>,
ids: &ReleaseIds,
claims: &NameClaims,
) -> Option<MovieMatch> {
if let Some(tmdb_id) = tmdb_id {
return unique(
wanted
.iter()
.filter(|movie| movie.tmdb_id == Some(tmdb_id))
.map(|movie| MovieMatch {
let imdb_id = ids.imdb_id.as_deref().and_then(imdb_key);
if ids.tmdb_id.is_some() || imdb_id.is_some() {
// A title matched by both ids is still one title, so each wanted
// title is offered at most once and only a genuine disagreement
// between the two ids reads as an ambiguity.
return unique(wanted.iter().filter_map(|movie| {
let kind = if ids.tmdb_id.is_some() && movie.tmdb_id == ids.tmdb_id {
MatchKind::TmdbId
} else if imdb_id.is_some() && movie.imdb_id.as_deref().and_then(imdb_key) == imdb_id {
MatchKind::ImdbId
} else {
return None;
};
Some(MovieMatch {
movie: movie.id,
kind: MatchKind::TmdbId,
}),
);
kind,
})
}));
}
// A season or episode tag makes this a TV release, whatever the title
@@ -88,6 +114,26 @@ pub fn match_movie(
)
}
/// The digits of an `IMDb` title id, or `None` when the string is not one.
///
/// Trackers spell the same id several ways: TMDB and most feeds write
/// `tt0317248`, some Torznab definitions emit the bare number, and either
/// may or may not carry the leading zeros `IMDb` pads to seven digits. All
/// of those are the same title, and nothing else is an id at all.
fn imdb_key(id: &str) -> Option<&str> {
let digits = id.trim();
let digits = digits
.strip_prefix("tt")
.or_else(|| digits.strip_prefix("TT"))
.unwrap_or(digits);
if !digits.bytes().all(|byte| byte.is_ascii_digit()) {
return None;
}
// An id of nothing but zeros is a placeholder, not a title.
let trimmed = digits.trim_start_matches('0');
(!trimmed.is_empty()).then_some(trimmed)
}
/// A release with no year matches only a title that has none either: films
/// share titles across decades, and the remake is not the one that is wanted.
fn years_agree(wanted: Option<u16>, release: Option<u16>) -> bool {
@@ -156,7 +202,7 @@ fn fold(character: char) -> Option<&'static str> {
#[cfg(test)]
mod tests {
use super::{match_key, match_movie, MatchKind, MovieMatch, WantedMovie};
use super::{match_key, match_movie, MatchKind, MovieMatch, ReleaseIds, WantedMovie};
use crate::MovieId;
fn wanted() -> Vec<WantedMovie> {
@@ -164,26 +210,44 @@ mod tests {
WantedMovie {
id: MovieId(1),
tmdb_id: Some(693_134),
imdb_id: Some("tt15239678".to_owned()),
title: "Dune: Part Two".to_owned(),
year: Some(2024),
},
WantedMovie {
id: MovieId(2),
tmdb_id: Some(438_631),
imdb_id: Some("tt1160419".to_owned()),
title: "Dune".to_owned(),
year: Some(2021),
},
WantedMovie {
id: MovieId(3),
tmdb_id: Some(194),
imdb_id: None,
title: "Amélie".to_owned(),
year: Some(2001),
},
]
}
fn ids(tmdb_id: Option<u32>, imdb_id: Option<&str>) -> ReleaseIds {
ReleaseIds {
tmdb_id,
imdb_id: imdb_id.map(ToOwned::to_owned),
}
}
fn matched(name: &str, tmdb_id: Option<u32>) -> Option<MovieMatch> {
match_movie(&wanted(), tmdb_id, &arr_parse::parse(name))
match_movie(&wanted(), &ids(tmdb_id, None), &arr_parse::parse(name))
}
fn matched_by_imdb(name: &str, imdb_id: &str) -> Option<MovieMatch> {
match_movie(
&wanted(),
&ids(None, Some(imdb_id)),
&arr_parse::parse(name),
)
}
#[test]
@@ -242,11 +306,12 @@ mod tests {
let wanted = vec![WantedMovie {
id: MovieId(4),
tmdb_id: Some(1),
imdb_id: None,
title: "Fallout".to_owned(),
year: Some(2024),
}];
let claims = arr_parse::parse("Fallout.2024.S01E03.1080p.WEB-DL");
assert_eq!(match_movie(&wanted, None, &claims), None);
assert_eq!(match_movie(&wanted, &ReleaseIds::default(), &claims), None);
}
#[test]
@@ -255,18 +320,20 @@ mod tests {
WantedMovie {
id: MovieId(5),
tmdb_id: Some(1),
imdb_id: None,
title: "The Thing".to_owned(),
year: Some(1982),
},
WantedMovie {
id: MovieId(6),
tmdb_id: Some(2),
imdb_id: None,
title: "The Thing".to_owned(),
year: Some(1982),
},
];
let claims = arr_parse::parse("The.Thing.1982.1080p.BluRay");
assert_eq!(match_movie(&wanted, None, &claims), None);
assert_eq!(match_movie(&wanted, &ReleaseIds::default(), &claims), None);
}
#[test]
@@ -275,4 +342,102 @@ mod tests {
assert_eq!(match_key("Fast and Furious"), "fast and furious");
assert_eq!(match_key(" Dune: Part Two "), "dune part two");
}
#[test]
fn a_supplied_imdb_id_matches_whatever_the_name_says() {
assert_eq!(
matched_by_imdb("Some.Scene.Name.2024.2160p.WEB-DL", "tt15239678"),
Some(MovieMatch {
movie: MovieId(1),
kind: MatchKind::ImdbId,
})
);
}
/// Torznab definitions spell the same id three ways. All of them are the
/// one title, and matching none of them is how the id lane stays unused.
#[test]
fn an_imdb_id_matches_however_the_tracker_spells_it() {
for spelling in ["tt15239678", "15239678", "tt0015239678", " tt15239678 "] {
assert_eq!(
matched_by_imdb("Some.Scene.Name.2024.2160p.WEB-DL", spelling),
Some(MovieMatch {
movie: MovieId(1),
kind: MatchKind::ImdbId,
}),
"{spelling}"
);
}
}
#[test]
fn a_supplied_imdb_id_that_is_not_wanted_ends_the_comparison() {
assert_eq!(
matched_by_imdb("Dune.Part.Two.2024.2160p.WEB-DL", "tt0000001"),
None
);
}
/// A placeholder or a non-id is no id at all, so the title lane still runs.
#[test]
fn a_junk_imdb_id_falls_through_to_the_title() {
for junk in ["", "tt", "tt0000000", "0", "n/a"] {
assert_eq!(
matched_by_imdb("Dune.Part.Two.2024.2160p.WEB-DL", junk),
Some(MovieMatch {
movie: MovieId(1),
kind: MatchKind::TitleAndYear,
}),
"{junk}"
);
}
}
/// Both ids on the same title is one match, not an ambiguity, and the
/// TMDB id is what reports it.
#[test]
fn two_ids_that_agree_are_one_match() {
assert_eq!(
match_movie(
&wanted(),
&ids(Some(693_134), Some("tt15239678")),
&arr_parse::parse("Some.Scene.Name.2024.2160p.WEB-DL"),
),
Some(MovieMatch {
movie: MovieId(1),
kind: MatchKind::TmdbId,
})
);
}
/// The tracker named two different films. Picking either is a coin toss
/// on which one gets grabbed.
#[test]
fn two_ids_that_disagree_match_nothing() {
assert_eq!(
match_movie(
&wanted(),
&ids(Some(693_134), Some("tt1160419")),
&arr_parse::parse("Dune.Part.Two.2024.2160p.WEB-DL"),
),
None
);
}
/// One id the library does not carry does not veto the other: a title
/// TMDB knows no `IMDb` id for is matched by its TMDB id as before.
#[test]
fn an_id_the_library_lacks_does_not_block_the_other() {
assert_eq!(
match_movie(
&wanted(),
&ids(Some(194), Some("tt0211915")),
&arr_parse::parse("Amelie.2001.1080p.BluRay"),
),
Some(MovieMatch {
movie: MovieId(3),
kind: MatchKind::TmdbId,
})
);
}
}
crates/arr-daemon/src/grab.rs
+8 -1
View File
@@ -180,28 +180,35 @@ impl GrabAction {
let original_language =
(!metadata.original_language.is_empty()).then_some(metadata.original_language.clone());
let digital_release = metadata.digital_release.map(|date| date.to_string());
// §6.2: the id RSS matching prefers, and the one Torznab movie
// searches take. TMDB does not know one for every title.
let imdb_id = metadata.imdb_id.clone();
let title_ref = title.as_str();
let original_language_ref = original_language.as_deref();
let digital_release_ref = digital_release.as_deref();
let imdb_id_ref = imdb_id.as_deref();
let changed = sqlx::query!(
r#"UPDATE movies
SET title = ?, year = ?, original_language = ?, digital_release = ?,
imdb_id = ?,
metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'),
search_attempts = 0, last_searched_at = NULL,
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
WHERE id = ? AND (
title IS NOT ? OR year IS NOT ? OR original_language IS NOT ?
OR digital_release IS NOT ?
OR digital_release IS NOT ? OR imdb_id IS NOT ?
)"#,
title_ref,
year,
original_language_ref,
digital_release_ref,
imdb_id_ref,
movie.id,
title_ref,
year,
original_language_ref,
digital_release_ref,
imdb_id_ref,
)
.execute(database.pool())
.await?
crates/arr-daemon/src/rss.rs
+9 -2
View File
@@ -14,7 +14,7 @@ use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::path::PathBuf;
use arr_core::matching::{match_movie, MatchKind, WantedMovie};
use arr_core::matching::{match_movie, MatchKind, ReleaseIds, WantedMovie};
use arr_core::{Language, MovieId};
use arr_db::{Blacklist, Db, MoviePolicy};
use arr_dl::TransmissionClient;
@@ -124,7 +124,11 @@ impl RssAction {
for release in releases {
let claims = arr_parse::parse(&release.name);
let Some(matched) = match_movie(wanted, release.tmdb_id, &claims) else {
let ids = ReleaseIds {
tmdb_id: release.tmdb_id,
imdb_id: release.imdb_id.clone(),
};
let Some(matched) = match_movie(wanted, &ids, &claims) else {
continue;
};
let movie_id = matched.movie.0;
@@ -142,6 +146,7 @@ impl RssAction {
release = release.name,
by = match matched.kind {
MatchKind::TmdbId => "tmdb id",
MatchKind::ImdbId => "imdb id",
MatchKind::TitleAndYear => "title and year",
},
"RSS result matched a wanted title"
@@ -218,6 +223,7 @@ async fn wanted_movies(database: &Db) -> Result<Vec<WantedMovie>, GrabError> {
r#"
SELECT m.id AS "id!: i64",
m.tmdb_id AS "tmdb_id!: i64",
m.imdb_id AS "imdb_id: String",
m.title AS "title!: String",
m.year
FROM movies m
@@ -242,6 +248,7 @@ async fn wanted_movies(database: &Db) -> Result<Vec<WantedMovie>, GrabError> {
.map(|row| WantedMovie {
id: MovieId(row.id),
tmdb_id: u32::try_from(row.tmdb_id).ok(),
imdb_id: row.imdb_id,
title: row.title,
year: row.year.and_then(|year| u16::try_from(year).ok()),
})
crates/arr-db/migrations/0013_movie_imdb_id.sql
+5
View File
@@ -0,0 +1,5 @@
-- RSS matching (§6.2) prefers an id the tracker supplied over anything read
-- off the release name, and `imdbid` is the id Prowlarr's definitions emit
-- most often. Nullable: TMDB does not know one for every title, and the
-- column is only filled by the metadata refresh.
ALTER TABLE movies ADD COLUMN imdb_id TEXT;
crates/arr-indexer/src/search.rs
+51
View File
@@ -72,6 +72,10 @@ pub struct SearchRelease {
/// TMDB id the tracker attached to the item, when it attached one. An ID
/// the indexer supplies beats anything read off the release name.
pub tmdb_id: Option<u32>,
/// `IMDb` id the tracker attached to the item, as it spelled it. Kept
/// verbatim because definitions differ on the `tt` prefix and on the
/// zero padding; matching (§6.2) normalises both sides.
pub imdb_id: Option<String>,
}
/// Results from one indexer in a multi-indexer search.
@@ -283,6 +287,7 @@ struct ReleaseBuilder {
link: Option<String>,
enclosure_url: Option<String>,
tmdb_id: Option<u32>,
imdb_id: Option<String>,
}
impl ReleaseBuilder {
@@ -300,6 +305,7 @@ impl ReleaseBuilder {
publish_date: self.publish_date,
download_url,
tmdb_id: self.tmdb_id,
imdb_id: self.imdb_id,
})
}
@@ -428,6 +434,12 @@ fn read_element_attributes(
.filter(|&id| id != 0)
.or(builder.tmdb_id);
}
Some("imdbid" | "imdb") => {
builder.imdb_id = value
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty())
.or_else(|| builder.imdb_id.take());
}
_ => {}
}
}
@@ -760,6 +772,45 @@ mod tests {
);
}
/// `imdbid` is the id most Cardigann definitions carry, and it is kept
/// exactly as the tracker spelled it — matching normalises (§6.2).
#[test]
fn a_torznab_imdb_id_is_kept() {
let body = br#"
<rss version="2.0" xmlns:torznab="http://torznab.com/schemas/2015/feed">
<channel>
<item>
<title>Dune.Part.Two.2024.2160p.WEB-DL</title>
<guid>prefixed</guid>
<link>https://indexer.invalid/download/prefixed</link>
<torznab:attr name="imdbid" value="tt15239678"/>
</item>
<item>
<title>Dune.2021.2160p.WEB-DL</title>
<guid>bare</guid>
<link>https://indexer.invalid/download/bare</link>
<torznab:attr name="imdbid" value=" 1160419 "/>
</item>
<item>
<title>Nosferatu.2024.2160p.WEB-DL</title>
<guid>anonymous</guid>
<link>https://indexer.invalid/download/anonymous</link>
<torznab:attr name="imdbid" value=""/>
</item>
</channel>
</rss>
"#;
let releases = parse_releases(3, body).expect("feed structure is valid");
assert_eq!(releases[0].imdb_id.as_deref(), Some("tt15239678"));
assert_eq!(releases[1].imdb_id.as_deref(), Some("1160419"));
assert_eq!(
releases[2].imdb_id, None,
"an empty attribute is the tracker saying it does not know"
);
}
/// A pack and one episode of that pack carry the same series title, so
/// the tag on the release name is the only thing separating them.
#[test]