diff --git a/.sqlx/query-0696aa0067ba82e31b5e9e46793080cd50793c8bb0bfee16ef932dcee2b85bf4.json b/.sqlx/query-0696aa0067ba82e31b5e9e46793080cd50793c8bb0bfee16ef932dcee2b85bf4.json new file mode 100644 index 0000000..6ffb27f --- /dev/null +++ b/.sqlx/query-0696aa0067ba82e31b5e9e46793080cd50793c8bb0bfee16ef932dcee2b85bf4.json @@ -0,0 +1,44 @@ +{ + "db_name": "SQLite", + "query": "SELECT id AS \"id!: i64\", path AS \"path!: String\", size AS \"size!: i64\", probed AS \"probed?: serde_json::Value\", json_extract(waiver, '$.rule') AS \"waiver?: String\" FROM media_files WHERE owner_kind = 'movie' AND owner_id = ? ORDER BY path", + "describe": { + "columns": [ + { + "name": "id!: i64", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "path!: String", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "size!: i64", + "ordinal": 2, + "type_info": "Integer" + }, + { + "name": "probed?: serde_json::Value", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "waiver?: String", + "ordinal": 4, + "type_info": "Null" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + true, + false, + false, + true, + null + ] + }, + "hash": "0696aa0067ba82e31b5e9e46793080cd50793c8bb0bfee16ef932dcee2b85bf4" +} diff --git a/.sqlx/query-48c46aa1908add3d6abff142dbb22c2bce92c6483564c159731e64fdd5f7f8bf.json b/.sqlx/query-071c14544e225001d31c5da60c90d3144fc5a071893c74b2c1eea51bfe00ac97.json similarity index 58% rename from .sqlx/query-48c46aa1908add3d6abff142dbb22c2bce92c6483564c159731e64fdd5f7f8bf.json rename to .sqlx/query-071c14544e225001d31c5da60c90d3144fc5a071893c74b2c1eea51bfe00ac97.json index 0683baa..14a5622 100644 --- a/.sqlx/query-48c46aa1908add3d6abff142dbb22c2bce92c6483564c159731e64fdd5f7f8bf.json +++ b/.sqlx/query-071c14544e225001d31c5da60c90d3144fc5a071893c74b2c1eea51bfe00ac97.json @@ -1,20 +1,26 @@ { "db_name": "SQLite", - "query": "SELECT normalised_name AS \"normalised_name!: String\" FROM blacklist", + "query": "SELECT normalised_name AS \"normalised_name!: String\", infohash FROM blacklist", "describe": { "columns": [ { "name": "normalised_name!: String", "ordinal": 0, "type_info": "Text" + }, + { + "name": "infohash", + "ordinal": 1, + "type_info": "Text" } ], "parameters": { "Right": 0 }, "nullable": [ - false + false, + true ] }, - "hash": "48c46aa1908add3d6abff142dbb22c2bce92c6483564c159731e64fdd5f7f8bf" + "hash": "071c14544e225001d31c5da60c90d3144fc5a071893c74b2c1eea51bfe00ac97" } diff --git a/.sqlx/query-7505ce51ba10ce886c5b6888e8d27aecea4ee2f5412894dfe99057e8e88c7db3.json b/.sqlx/query-7505ce51ba10ce886c5b6888e8d27aecea4ee2f5412894dfe99057e8e88c7db3.json new file mode 100644 index 0000000..57c0166 --- /dev/null +++ b/.sqlx/query-7505ce51ba10ce886c5b6888e8d27aecea4ee2f5412894dfe99057e8e88c7db3.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT count(*) AS \"count!: i64\" FROM blacklist\n WHERE normalised_name = ? AND infohash IS NULL", + "describe": { + "columns": [ + { + "name": "count!: i64", + "ordinal": 0, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + }, + "hash": "7505ce51ba10ce886c5b6888e8d27aecea4ee2f5412894dfe99057e8e88c7db3" +} diff --git a/.sqlx/query-b4b63449fdfb0bf5fb7095577d2e4759784a92ad87cb151150689fc355473328.json b/.sqlx/query-7d56b343c398ed8dee3a1c1f7cc8464b2529c83ec04453cc604ab064e0456194.json similarity index 54% rename from .sqlx/query-b4b63449fdfb0bf5fb7095577d2e4759784a92ad87cb151150689fc355473328.json rename to .sqlx/query-7d56b343c398ed8dee3a1c1f7cc8464b2529c83ec04453cc604ab064e0456194.json index 72d9092..2d94bac 100644 --- a/.sqlx/query-b4b63449fdfb0bf5fb7095577d2e4759784a92ad87cb151150689fc355473328.json +++ b/.sqlx/query-7d56b343c398ed8dee3a1c1f7cc8464b2529c83ec04453cc604ab064e0456194.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "INSERT INTO blacklist (infohash, normalised_name, reason)\n VALUES (?, ?, ?)\n ON CONFLICT (infohash) DO NOTHING", + "query": "INSERT INTO blacklist (infohash, normalised_name, reason)\n VALUES (?, ?, ?)\n ON CONFLICT (infohash) DO NOTHING", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "b4b63449fdfb0bf5fb7095577d2e4759784a92ad87cb151150689fc355473328" + "hash": "7d56b343c398ed8dee3a1c1f7cc8464b2529c83ec04453cc604ab064e0456194" } diff --git a/.sqlx/query-8618f066558095607d224389ea16b901e1fd55c3e3b71d61433ddd9e2fa9bbf0.json b/.sqlx/query-8618f066558095607d224389ea16b901e1fd55c3e3b71d61433ddd9e2fa9bbf0.json new file mode 100644 index 0000000..1e15cba --- /dev/null +++ b/.sqlx/query-8618f066558095607d224389ea16b901e1fd55c3e3b71d61433ddd9e2fa9bbf0.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE releases SET verdict = 'rejected', rejected_rule = ? WHERE id = ?", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "8618f066558095607d224389ea16b901e1fd55c3e3b71d61433ddd9e2fa9bbf0" +} diff --git a/Cargo.lock b/Cargo.lock index c6fb5df..e5bb9b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,6 +110,7 @@ name = "arr-db" version = "0.1.0" dependencies = [ "arr-core", + "arr-parse", "serde", "serde_json", "sqlx", diff --git a/crates/arr-api/src/lib.rs b/crates/arr-api/src/lib.rs index 98fc8a9..8a5bc90 100644 --- a/crates/arr-api/src/lib.rs +++ b/crates/arr-api/src/lib.rs @@ -22,7 +22,9 @@ use utoipa_axum::routes; use utoipa_scalar::{Scalar, Servable}; pub use health::{Check, Health, HealthReport, Status}; -pub use movies::{Accepted, AttentionQueues, CreateMovie, ErrorBody, Movie, Release, UpdateMovie}; +pub use movies::{ + Accepted, AttentionQueues, CreateMovie, ErrorBody, Movie, MovieFile, Release, UpdateMovie, +}; pub use owners::{CreateOwner, Owner, UpdateOwner}; pub use roots::Root; pub use search::{ClassifiedRelease, SearchResponse}; @@ -67,6 +69,7 @@ fn api_router() -> OpenApiRouter { .routes(routes!(movies::get, movies::update, movies::delete)) .routes(routes!(movies::search)) .routes(routes!(movies::releases)) + .routes(routes!(movies::files)) .routes(routes!(movies::grab)) .routes(routes!(movies::attention)) .routes(routes!(movies::list_owners)) diff --git a/crates/arr-api/src/movies.rs b/crates/arr-api/src/movies.rs index 0e7b098..a8331d1 100644 --- a/crates/arr-api/src/movies.rs +++ b/crates/arr-api/src/movies.rs @@ -69,6 +69,23 @@ pub struct Release { pub rejected_rule: Option, } +/// A library file and what it cost to accept it (`DESIGN.md` §5.7). +/// +/// `waiver` names the rule that was relaxed to let a soft fail in. It is the +/// difference between a file that satisfies the policy and one that merely +/// plays, so it travels with the file everywhere the file does. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct MovieFile { + pub id: i64, + pub path: String, + pub size: i64, + /// What `ffprobe` found (§5.6). + pub probed: Option, + /// The relaxed rule's name, or `null` for a clean import. Shares its + /// vocabulary with `Release::rejected_rule`. + pub waiver: Option, +} + #[derive(Debug, Clone, Serialize, ToSchema)] pub struct AttentionQueues { pub no_pt_source: Vec, @@ -385,6 +402,27 @@ pub async fn releases( Ok(Json(releases)) } +#[utoipa::path( + get, path = "/api/movies/{movie_id}/files", tag = "movies", + params(("movie_id" = i64, Path, description = "Movie row id")), + responses( + (status = 200, body = [MovieFile]), + (status = 404, body = ErrorBody), + (status = 500, body = ErrorBody), + (status = 503, body = ErrorBody) + ) +)] +pub async fn files( + State(state): State, + Path(id): Path, +) -> Result>, ApiError> { + load_movie(&state, id).await?; + let files = sqlx::query_as!(MovieFile, r#"SELECT id AS "id!: i64", path AS "path!: String", size AS "size!: i64", probed AS "probed?: serde_json::Value", json_extract(waiver, '$.rule') AS "waiver?: String" FROM media_files WHERE owner_kind = 'movie' AND owner_id = ? ORDER BY path"#, id) + .fetch_all(pool(&state)?) + .await?; + Ok(Json(files)) +} + #[utoipa::path( post, path = "/api/movies/{movie_id}/releases/{release_id}/grab", tag = "movies", params(("movie_id" = i64, Path), ("release_id" = i64, Path)), @@ -563,6 +601,39 @@ mod tests { response.json().await.expect("movie json") } + /// §5.7: a soft-failed import is imported and waived, and the waiver + /// reaches the API — a file that merely plays must never read as a clean + /// match. + #[tokio::test] + async fn a_movie_file_carries_its_waiver() { + let (_dir, state, base) = application().await; + let movie = add_movie(&base, 693_134, 1).await; + let movie_id = movie["id"].as_i64().expect("movie id"); + let pool = state.database().expect("database").pool(); + sqlx::query( + r#"INSERT INTO media_files (owner_kind, owner_id, path, size, probed, waiver) + VALUES ('movie', ?, '/library/dune.mkv', 23622320128, + '{"resolution":"1080p"}', '{"rule":"required_audio"}')"#, + ) + .bind(movie_id) + .execute(pool) + .await + .expect("media file"); + + let files: Vec = + reqwest::get(format!("{base}/api/movies/{movie_id}/files")) + .await + .expect("files") + .json() + .await + .expect("json"); + + assert_eq!(files.len(), 1); + assert_eq!(files[0]["path"], "/library/dune.mkv"); + assert_eq!(files[0]["waiver"], "required_audio"); + assert_eq!(files[0]["probed"]["resolution"], "1080p"); + } + #[tokio::test] async fn crud_preserves_intent_and_overrides() { let (_dir, _state, base) = application().await; diff --git a/crates/arr-api/src/search.rs b/crates/arr-api/src/search.rs index 410bd15..a9ecc38 100644 --- a/crates/arr-api/src/search.rs +++ b/crates/arr-api/src/search.rs @@ -4,6 +4,7 @@ use arr_core::policy::{evaluate, Candidate}; use arr_core::score::score; use arr_core::{Language, Policy, Rule, TitleOverrides, Verdict}; use arr_db::policy::language; +use arr_db::{blacklist, Blacklist}; use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest, TvSelector, TvTarget}; use axum::extract::{Query, State}; use axum::Json; @@ -261,6 +262,7 @@ async fn movie_releases( .indexers() .await .map_err(|_| ApiError::Unavailable)?; + let blacklist = Blacklist::load(database.pool()).await?; let policy = loaded.policy; let overrides = loaded.overrides; let original_language = title_language( @@ -291,7 +293,13 @@ async fn movie_releases( match prowlarr.search_indexer(indexer.id, &indexer_request).await { Ok(releases) => { for release in releases { - classified.push(classify(release, &policy, &overrides, &original_language)?); + classified.push(classify( + release, + &policy, + &overrides, + &original_language, + &blacklist, + )?); } } Err(error) => { @@ -350,6 +358,7 @@ async fn episode_releases( .indexers() .await .map_err(|_| ApiError::Unavailable)?; + let blacklist = Blacklist::load(database.pool()).await?; let mut classified = Vec::new(); for indexer in indexers { @@ -364,6 +373,7 @@ async fn episode_releases( &loaded.policy, &loaded.overrides, &original_language, + &blacklist, )?); } } @@ -446,11 +456,17 @@ fn upstream_error(error: &arr_meta::Error) -> ApiError { } } +/// Classify one release for the §9.3 buckets. +/// +/// A blacklisted release (§6.3) is rejected whatever the policy makes of its +/// name, and says so: the operator sees why it is not offered rather than a +/// row that looks grabbable and silently is not. fn classify( release: SearchRelease, policy: &Policy, overrides: &TitleOverrides, original_language: &Language, + blacklist: &Blacklist, ) -> Result { let parsed = arr_parse::parse(&release.name); let evaluation = evaluate( @@ -460,7 +476,11 @@ fn classify( Candidate::PreGrab(&parsed), release.size, ); - let (verdict, rule) = verdict(&evaluation.verdict); + let (verdict, rule) = if blacklist.blocks_candidate(&release.name, &release.download_url) { + ("rejected", Some(blacklist::RULE.to_owned())) + } else { + verdict(&evaluation.verdict) + }; let score = score( policy, Candidate::PreGrab(&parsed), @@ -686,6 +706,30 @@ mod tests { .find(|release| release["guid"] == "good") .expect("eligible"); assert_eq!(eligible["score"], 0); + + // §6.3: once that release has hard-failed, the manual view must not + // keep offering it as a clean match — it is rejected, and says why. + blacklist::add( + state.database().expect("database").pool(), + None, + "Dune.Part.Two.2024.2160p.WEB-DL", + "dolby_vision_profile", + ) + .await + .expect("blacklist"); + let releases: Vec = + reqwest::get(format!("{base}/api/releases?movie_id=1")) + .await + .expect("releases") + .json() + .await + .expect("json"); + let blacklisted = releases + .iter() + .find(|release| release["guid"] == "good") + .expect("blacklisted"); + assert_eq!(blacklisted["verdict"], "rejected"); + assert_eq!(blacklisted["rule"], "blacklisted"); } #[tokio::test] @@ -796,6 +840,7 @@ mod tests { &policy, &TitleOverrides::default(), &Language::Other("en".into()), + &Blacklist::default(), ) .expect("classified release"); diff --git a/crates/arr-daemon/src/grab.rs b/crates/arr-daemon/src/grab.rs index 9c0de1b..fcac3f6 100644 --- a/crates/arr-daemon/src/grab.rs +++ b/crates/arr-daemon/src/grab.rs @@ -13,13 +13,13 @@ //! - the `grabs` row is written from that response, so a crash between the add //! and the insert heals on the next tick instead of leaving an orphan. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::path::PathBuf; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use arr_core::policy::{evaluate, Candidate}; use arr_core::{score::score, Language, Policy, TitleOverrides, Verdict}; -use arr_db::{Db, MoviePolicy}; +use arr_db::{blacklist, Blacklist, Db, MoviePolicy}; use arr_dl::{AddTorrent, TorrentSource, TransmissionClient}; use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest}; @@ -167,7 +167,7 @@ impl GrabAction { for grab in sent { let Some(progress) = torrents.get(&grab.infohash.to_ascii_lowercase()) else { // Gone from Transmission. Deciding whether that is a failure - // or a manual removal is issue #24's; leaving the row alone + // or a manual removal is issue #86's; leaving the row alone // keeps this tick from re-grabbing behind the operator. continue; }; @@ -252,6 +252,7 @@ impl GrabAction { indexers: &[i64], loaded: &MoviePolicy, original_language: &Language, + blacklist: &Blacklist, ) -> Result, GrabError> { let request = SearchRequest::Text { query: search_query(movie), @@ -274,6 +275,7 @@ impl GrabAction { &loaded.policy, &loaded.overrides, original_language, + blacklist, ) .await?; if let Some(candidate) = stored { @@ -314,17 +316,24 @@ impl GrabAction { }; let original_language = arr_db::policy::language(original_language); - let candidates = self - .search(database, movie, indexers, &loaded, &original_language) - .await?; // §6.3: anything that hard-failed post-ffprobe is never grabbed - // again. The same release reappears under new infohashes, so the key - // is the normalised name. - let blacklisted = blacklisted_names(database).await?; - let Some(winner) = candidates - .into_iter() - .find(|candidate| !blacklisted.contains(&arr_parse::normalise(&candidate.name))) - else { + // again. `store_release` already rejects a blacklisted candidate as + // it classifies it, so this second pass only catches a row that was + // classified before the blacklist entry existed. + let blacklist = Blacklist::load(database.pool()).await?; + let candidates = self + .search( + database, + movie, + indexers, + &loaded, + &original_language, + &blacklist, + ) + .await?; + let Some(winner) = candidates.into_iter().find(|candidate| { + !blacklist.blocks_candidate(&candidate.name, &candidate.download_url) + }) else { tracing::info!( movie_id = movie.id, title = movie.title, @@ -345,6 +354,15 @@ impl GrabAction { .await?; let infohash = added.hash.to_ascii_lowercase(); + // §6.3's second key. A `.torrent` link hides its infohash until + // Transmission has fetched it, so the same blacklisted torrent can + // reach here under a new name. + if blacklist.blocks_infohash(&infohash) { + self.drop_blacklisted_torrent(database, movie, &winner, &added) + .await?; + return Ok(None); + } + // A duplicate here is the restart case: the torrent was added before // the process died. `DO NOTHING` keeps the original row. let inserted = sqlx::query!( @@ -390,6 +408,60 @@ impl GrabAction { format!("grabbed {} as grab {}", winner.name, inserted.id), ))) } + + /// Undo a grab whose infohash turned out to be blacklisted (§6.3). + /// + /// The name is added to the blacklist so the next tick stops at the cheap + /// check instead of paying Transmission again, and no `grabs` row is + /// written, which leaves the title a gap for the next candidate. + async fn drop_blacklisted_torrent( + &self, + database: &Db, + movie: &PendingMovie, + winner: &Eligible, + added: &arr_dl::AddedTorrent, + ) -> Result<(), GrabError> { + let release_name = &winner.name; + blacklist::add( + database.pool(), + None, + release_name, + "blacklisted infohash under a new name", + ) + .await?; + // `store_release` classified this row before the infohash was known, + // so it still reads eligible. Correct it here rather than waiting for + // the next search to overwrite it: until then §9.3's manual view + // would keep offering a release this tick just refused. + sqlx::query!( + "UPDATE releases SET verdict = 'rejected', rejected_rule = ? WHERE id = ?", + blacklist::RULE, + winner.id + ) + .execute(database.pool()) + .await?; + if added.was_duplicate { + // The earlier grab's torrent, still working off its seeding + // obligation (§7.3). Nothing here deletes a torrent. + tracing::warn!( + movie_id = movie.id, + release = release_name, + infohash = added.hash, + "blacklisted torrent re-listed under a new name; left seeding" + ); + } else { + // This tick added it seconds ago, so it carries no seeding + // obligation and has nothing on disk worth keeping. + self.transmission.remove_torrent(added.id, true).await?; + tracing::warn!( + movie_id = movie.id, + release = release_name, + infohash = added.hash, + "blacklisted torrent re-listed under a new name; removed" + ); + } + Ok(()) + } } impl Action for GrabAction { @@ -471,6 +543,11 @@ async fn pending_movies(database: &Db) -> Result, GrabError> { /// /// Returns the candidate only when the release is eligible: automatic /// selection never takes a waiver (§9.3). +/// +/// The blacklist is applied here rather than at selection so it reaches every +/// trigger (§6.3: "including by RSS") and so the stored row says why — a +/// blacklisted release is rejected under the `blacklisted` rule, which is +/// what stops §9.3's manual view from offering it as a clean match. async fn store_release( database: &Db, movie_id: i64, @@ -478,6 +555,7 @@ async fn store_release( policy: &Policy, overrides: &TitleOverrides, original_language: &Language, + blacklist: &Blacklist, ) -> Result, GrabError> { let parsed = arr_parse::parse(&release.name); let evaluation = evaluate( @@ -501,7 +579,11 @@ async fn store_release( } else { scored.source.saturating_add(scored.seeders) }; - let (verdict, rule) = verdict_columns(&evaluation.verdict); + let (verdict, rule) = if blacklist.blocks_candidate(&release.name, &release.download_url) { + ("rejected", Some(blacklist::RULE.to_owned())) + } else { + verdict_columns(&evaluation.verdict) + }; let parsed_json = serde_json::to_string(&parsed).map_err(|source| GrabError::Parsed { name: release.name.clone(), source, @@ -565,17 +647,6 @@ async fn store_release( })) } -/// Every blacklist key (§6.3), for filtering candidates. Household scale: a -/// handful of rows, refetched per title rather than cached. -async fn blacklisted_names(database: &Db) -> Result, GrabError> { - let names = sqlx::query_scalar!( - r#"SELECT normalised_name AS "normalised_name!: String" FROM blacklist"# - ) - .fetch_all(database.pool()) - .await?; - Ok(names.into_iter().collect()) -} - /// Record that the title was searched, so the next tick takes a different one. async fn record_search(database: &Db, movie_id: i64) -> Result<(), GrabError> { sqlx::query!( @@ -730,6 +801,17 @@ mod tests { .collect(); success(&json!({"torrents": torrents})) } + "torrent-remove" => { + let removed: Vec = arguments["ids"] + .as_array() + .map(|ids| ids.iter().filter_map(serde_json::Value::as_i64).collect()) + .unwrap_or_default(); + self.torrents + .lock() + .unwrap() + .retain(|torrent| !removed.contains(&torrent.id)); + success(&json!({})) + } _ => success(&json!({})), } } @@ -1022,6 +1104,75 @@ mod tests { ); } + /// §6.3 reaches every trigger, so the exclusion is applied where a + /// release is classified rather than where one is picked: the cached row + /// says `blacklisted`, which is also what stops §9.3's manual view from + /// offering it as a clean match. + #[tokio::test] + async fn a_blacklisted_release_is_cached_as_rejected() { + let (_dir, database) = wanted_movie().await; + blacklist::add( + database.pool(), + Some("ffff"), + "Dune.Part.Two.2024.2160p.WEB-DL.DDP5.1.Atmos", + "dolby_vision_profile", + ) + .await + .unwrap(); + let indexer = prowlarr().await; + let (downloader, _fake) = transmission().await; + + action(&indexer, &downloader).tick(&database).await.unwrap(); + + let (verdict, rule): (String, Option) = + sqlx::query_as("SELECT verdict, rejected_rule FROM releases WHERE guid = 'good'") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(verdict, "rejected"); + assert_eq!(rule.as_deref(), Some("blacklisted")); + } + + /// §6.3's second key. A `.torrent` link hides its infohash until + /// Transmission has fetched it, so the blacklisted torrent is only + /// recognised after the add — and must not leave a grab behind. + #[tokio::test] + async fn a_blacklisted_infohash_never_becomes_a_grab() { + let (_dir, database) = wanted_movie().await; + // What the fake hands back for the first torrent it accepts. + blacklist::add( + database.pool(), + Some(&format!("{:040x}", 7)), + "Some.Older.Name.Of.The.Same.Torrent", + "required_audio", + ) + .await + .unwrap(); + let indexer = prowlarr().await; + let (downloader, fake) = transmission().await; + + action(&indexer, &downloader).tick(&database).await.unwrap(); + + assert!(grabs(&database).await.is_empty()); + assert!( + fake.torrents().is_empty(), + "a torrent added this tick and then found blacklisted is removed" + ); + // Recorded under its new name, so the next tick stops before paying + // Transmission again. + let blacklist = Blacklist::load(database.pool()).await.unwrap(); + assert!(blacklist.blocks_name("Dune.Part.Two.2024.2160p.WEB-DL.DDP5.1.Atmos")); + // And the cached row stops reading eligible straight away, so §9.3's + // manual view never offers what this tick just refused. + let (verdict, rule): (String, Option) = + sqlx::query_as("SELECT verdict, rejected_rule FROM releases WHERE guid = 'good'") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(verdict, "rejected"); + assert_eq!(rule.as_deref(), Some("blacklisted")); + } + /// §6.3: `blocked` stops targeted search for a title. #[tokio::test] async fn a_blocked_title_is_not_searched() { diff --git a/crates/arr-daemon/src/import.rs b/crates/arr-daemon/src/import.rs index 283621f..1023549 100644 --- a/crates/arr-daemon/src/import.rs +++ b/crates/arr-daemon/src/import.rs @@ -203,7 +203,7 @@ impl ImportAction { let Some(content) = self.transmission.torrent_content(&pending.infohash).await? else { // Gone from Transmission. Whether that is a failure or a manual - // removal is issue #24's call; leave the grab alone. + // removal is issue #86's call; leave the grab alone. tracing::warn!( grab_id = pending.grab_id, infohash = pending.infohash, @@ -312,16 +312,12 @@ impl ImportAction { pending: &PendingImport, reason: &str, ) -> Result { - let normalised = arr_parse::normalise(&pending.release_name); - sqlx::query!( - "INSERT INTO blacklist (infohash, normalised_name, reason) - VALUES (?, ?, ?) - ON CONFLICT (infohash) DO NOTHING", - pending.infohash, - normalised, - reason + arr_db::blacklist::add( + database.pool(), + Some(&pending.infohash), + &pending.release_name, + reason, ) - .execute(database.pool()) .await?; sqlx::query!( "UPDATE grabs SET state = 'failed' WHERE id = ?", @@ -770,6 +766,60 @@ mod tests { ); } + /// §5.7 soft fail: watchable but not what was asked. It imports, and the + /// row carries the relaxed rule — the file must never read as a clean + /// match. The torrent is untouched either way (§7.3). + #[tokio::test] + async fn an_english_only_kids_import_carries_a_waiver() { + let h = harness(HDR10_PROBE).await; + // The kids policy requires Portuguese audio; `allow_english_audio` + // turns that hard fail into a waiver (§5.2, §5.7). + let kids_library = h.library.join("kids"); + std::fs::create_dir_all(&kids_library).unwrap(); + sqlx::query("UPDATE roots SET path = ? WHERE kind = 'movie' AND audience = 'kids'") + .bind(kids_library.to_string_lossy().into_owned()) + .execute(h.database.pool()) + .await + .unwrap(); + sqlx::query( + r#"UPDATE movies + SET root_id = (SELECT id FROM roots + WHERE kind = 'movie' AND audience = 'kids'), + overrides = '{"allow_english_audio":true}' + WHERE id = 1"#, + ) + .execute(h.database.pool()) + .await + .unwrap(); + + let outcomes = h.action.tick(&h.database).await.unwrap(); + + assert_eq!(outcomes.len(), 1); + let waiver: Option = sqlx::query_scalar( + "SELECT json_extract(waiver, '$.rule') FROM media_files + WHERE owner_kind = 'movie' AND owner_id = 1", + ) + .fetch_one(h.database.pool()) + .await + .unwrap(); + assert_eq!(waiver.as_deref(), Some("required_audio")); + + let blacklisted: i64 = sqlx::query_scalar("SELECT count(*) FROM blacklist") + .fetch_one(h.database.pool()) + .await + .unwrap(); + assert_eq!(blacklisted, 0, "a soft fail blacklists nothing"); + let grab_state: String = sqlx::query_scalar("SELECT state FROM grabs") + .fetch_one(h.database.pool()) + .await + .unwrap(); + assert_eq!(grab_state, "imported"); + assert!( + h.downloads.join("Dune/Dune.mkv").is_file(), + "§7.3: neither failure mode deletes the torrent" + ); + } + /// §8: killed between the hardlink and the bookkeeping, a restart /// converges instead of failing on the existing destination. #[tokio::test] diff --git a/crates/arr-db/Cargo.toml b/crates/arr-db/Cargo.toml index bdb3a89..5d71524 100644 --- a/crates/arr-db/Cargo.toml +++ b/crates/arr-db/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] arr-core = { workspace = true } +arr-parse = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } diff --git a/crates/arr-db/src/blacklist.rs b/crates/arr-db/src/blacklist.rs new file mode 100644 index 0000000..989ab04 --- /dev/null +++ b/crates/arr-db/src/blacklist.rs @@ -0,0 +1,225 @@ +//! The release blacklist (`DESIGN.md` §6.3). +//! +//! Anything that hard-failed post-`ffprobe` (§5.7) is never grabbed again, +//! *including by RSS*. That "including" is why the check lives here rather +//! than inside the grab loop: every path that classifies a release — targeted +//! search, RSS matching, the manual search view — loads the same set and +//! rejects against it, so a new trigger cannot forget the rule. +//! +//! Two keys, because a release escapes either one alone: +//! +//! - **normalised name**, because the same release is re-uploaded under a new +//! infohash, and +//! - **infohash**, because the same torrent is re-listed under a new name. + +use std::collections::HashSet; + +use sqlx::SqlitePool; + +/// The rule name a blacklisted release is rejected under, shared by +/// `releases.rejected_rule` and the manual-search API so §9.3's "every +/// rejected row names the rule that killed it" reads the same everywhere. +pub const RULE: &str = "blacklisted"; + +/// Every blacklist key, loaded once per tick or request. +/// +/// Household scale: a handful of rows. Loading it whole and matching in +/// memory keeps the check identical for a release that has a database row and +/// one that has only just arrived from an indexer. +#[derive(Debug, Clone, Default)] +pub struct Blacklist { + names: HashSet, + infohashes: HashSet, +} + +impl Blacklist { + /// Read every blacklist row. + /// + /// # Errors + /// + /// If the query fails. + pub async fn load(pool: &SqlitePool) -> Result { + let rows = sqlx::query!( + r#"SELECT normalised_name AS "normalised_name!: String", infohash FROM blacklist"# + ) + .fetch_all(pool) + .await?; + + let mut blacklist = Self::default(); + for row in rows { + blacklist.names.insert(row.normalised_name); + if let Some(infohash) = row.infohash { + blacklist.infohashes.insert(infohash.to_ascii_lowercase()); + } + } + Ok(blacklist) + } + + /// Whether this release name has been blacklisted, under any spelling. + #[must_use] + pub fn blocks_name(&self, release_name: &str) -> bool { + self.names.contains(&arr_parse::normalise(release_name)) + } + + /// Whether this infohash has been blacklisted. Case-insensitive: + /// Transmission and Torznab disagree on the hex casing. + #[must_use] + pub fn blocks_infohash(&self, infohash: &str) -> bool { + self.infohashes.contains(&infohash.to_ascii_lowercase()) + } + + /// Whether a candidate is blacklisted before anything is sent to the + /// download client — by name, or by the infohash a magnet link carries. + /// + /// A `.torrent` URL hides its infohash until the download client has + /// fetched it, so a non-magnet candidate is judged on its name alone + /// here; [`Blacklist::blocks_infohash`] catches the rest post-add. + #[must_use] + pub fn blocks_candidate(&self, release_name: &str, download_url: &str) -> bool { + self.blocks_name(release_name) + || magnet_infohash(download_url).is_some_and(|hash| self.blocks_infohash(&hash)) + } +} + +/// Record a hard fail (§5.7). +/// +/// Both keys are written when both are known. The insert is idempotent: a +/// second hard fail of the same torrent keeps the first reason, and a re-run +/// after a crash between the blacklist and the grab update converges. +/// +/// # Errors +/// +/// If the insert fails. +pub async fn add( + pool: &SqlitePool, + infohash: Option<&str>, + release_name: &str, + reason: &str, +) -> Result<(), sqlx::Error> { + let infohash = infohash.map(str::to_ascii_lowercase); + let normalised = arr_parse::normalise(release_name); + // The name key has no unique index — the same name legitimately arrives + // under several infohashes — so a nameless duplicate is filtered here + // rather than by the schema. + let already = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!: i64" FROM blacklist + WHERE normalised_name = ? AND infohash IS NULL"#, + normalised + ) + .fetch_one(pool) + .await?; + if infohash.is_none() && already > 0 { + return Ok(()); + } + sqlx::query!( + "INSERT INTO blacklist (infohash, normalised_name, reason) + VALUES (?, ?, ?) + ON CONFLICT (infohash) DO NOTHING", + infohash, + normalised, + reason + ) + .execute(pool) + .await?; + Ok(()) +} + +/// The infohash a magnet link declares, lowercased. +/// +/// Only the 40-character hex form is recognised. Base32 `btih` values exist +/// in the wild but Transmission normalises them away, and guessing wrong here +/// would blacklist an unrelated release. +#[must_use] +pub fn magnet_infohash(download_url: &str) -> Option { + if !download_url.starts_with("magnet:") { + return None; + } + download_url + .split(['?', '&']) + .filter_map(|parameter| parameter.strip_prefix("xt=urn:btih:")) + .find(|hash| hash.len() == 40 && hash.chars().all(|c| c.is_ascii_hexdigit())) + .map(str::to_ascii_lowercase) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::{add, magnet_infohash, Blacklist}; + use crate::Db; + + const HASH: &str = "0123456789abcdef0123456789abcdef01234567"; + + async fn database() -> (Db, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let database = Db::connect(dir.path().join("test.db")).await.unwrap(); + database.migrate().await.unwrap(); + (database, dir) + } + + #[tokio::test] + async fn both_keys_are_recorded_and_matched() { + let (database, _dir) = database().await; + add( + database.pool(), + Some(&HASH.to_ascii_uppercase()), + "Dune.Part.Two.2024.2160p.WEB-DL.DV", + "dolby_vision_profile", + ) + .await + .unwrap(); + + let blacklist = Blacklist::load(database.pool()).await.unwrap(); + // §6.3: the same release under different separators is one key. + assert!(blacklist.blocks_name("Dune Part Two 2024 2160p WEB-DL DV")); + assert!(blacklist.blocks_infohash(HASH)); + assert!(!blacklist.blocks_name("Dune Part Two 2024 1080p WEB-DL")); + } + + #[tokio::test] + async fn a_second_hard_fail_of_the_same_torrent_adds_no_row() { + let (database, _dir) = database().await; + for _ in 0..2 { + add(database.pool(), Some(HASH), "Some.Release.2024", "size") + .await + .unwrap(); + } + add(database.pool(), None, "Some.Release.2024", "size") + .await + .unwrap(); + add(database.pool(), None, "Some.Release.2024", "size") + .await + .unwrap(); + + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM blacklist") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 2, "one row per key, not one per failure"); + } + + #[tokio::test] + async fn a_magnet_candidate_is_blocked_by_its_infohash_alone() { + let (database, _dir) = database().await; + add( + database.pool(), + Some(HASH), + "Old.Name.2024", + "required_audio", + ) + .await + .unwrap(); + let blacklist = Blacklist::load(database.pool()).await.unwrap(); + + let magnet = format!("magnet:?xt=urn:btih:{HASH}&dn=New.Name.2024"); + assert!(blacklist.blocks_candidate("New.Name.2024", &magnet)); + assert!(!blacklist.blocks_candidate("New.Name.2024", "https://tracker/x.torrent")); + } + + #[test] + fn magnet_infohashes_are_read_only_in_the_hex_form() { + let magnet = format!("magnet:?xt=urn:btih:{}&dn=x", HASH.to_ascii_uppercase()); + assert_eq!(magnet_infohash(&magnet), Some(HASH.to_owned())); + assert_eq!(magnet_infohash("https://tracker/x.torrent"), None); + assert_eq!(magnet_infohash("magnet:?xt=urn:btih:ZZZZ&dn=x"), None); + } +} diff --git a/crates/arr-db/src/lib.rs b/crates/arr-db/src/lib.rs index 173ef81..90da074 100644 --- a/crates/arr-db/src/lib.rs +++ b/crates/arr-db/src/lib.rs @@ -5,8 +5,10 @@ use std::path::Path; +pub mod blacklist; pub mod policy; +pub use blacklist::Blacklist; pub use policy::{MoviePolicy, PolicyColumns, PolicyError, TitlePolicy}; use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous};