diff --git a/Cargo.lock b/Cargo.lock index 4ea3fdd..595a426 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,8 +205,15 @@ name = "arr-subs" version = "0.1.0" dependencies = [ "arr-core", + "arr-parse", + "reqwest", + "serde", + "serde_json", + "tempfile", "thiserror", "tokio", + "tracing", + "wiremock", ] [[package]] diff --git a/crates/arr-subs/Cargo.toml b/crates/arr-subs/Cargo.toml index bcabaff..83a6dc6 100644 --- a/crates/arr-subs/Cargo.toml +++ b/crates/arr-subs/Cargo.toml @@ -19,10 +19,17 @@ translate-command = [] [dependencies] arr-core.workspace = true +arr-parse.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true thiserror.workspace = true +tracing.workspace = true [dev-dependencies] +tempfile.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +wiremock.workspace = true [lints] workspace = true diff --git a/crates/arr-subs/src/lib.rs b/crates/arr-subs/src/lib.rs index 1b6ade9..4e5ce27 100644 --- a/crates/arr-subs/src/lib.rs +++ b/crates/arr-subs/src/lib.rs @@ -22,6 +22,7 @@ use std::{fmt, future::Future, pin::Pin}; pub mod error; pub mod model; +pub mod opensubtitles; pub mod srt; pub mod translate; @@ -29,9 +30,16 @@ pub use error::{Error, Result}; pub use model::{ Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat, }; +pub use opensubtitles::{moviehash, OpenSubtitles, OpenSubtitlesConfig}; pub use srt::Cue; pub use translate::{Backend, BackendId, Batch, BatchCue, TranslateFuture, TranslatedCue}; +// The OpenSubtitles provider's HTTP paths are exercised by the integration +// tests, which link this crate; the unit-test target still has to satisfy +// the unused-crate-dependencies lint for the dev-dependency itself. +#[cfg(test)] +use wiremock as _; + /// The candidates one search turned up. pub type SearchFuture<'a> = Pin>> + Send + 'a>>; diff --git a/crates/arr-subs/src/opensubtitles.rs b/crates/arr-subs/src/opensubtitles.rs new file mode 100644 index 0000000..a456111 --- /dev/null +++ b/crates/arr-subs/src/opensubtitles.rs @@ -0,0 +1,723 @@ +//! The OpenSubtitles.com provider (`DESIGN.md` §15, issue #187). +//! +//! One of the two providers §15 names, and the primary one: it has an +//! official REST API, it keeps pt-PT and pt-BR apart properly, and it is the +//! only provider that answers `moviehash` searches — the signal that wins +//! ranking outright. +//! +//! Both lanes feed one search: `moviehash`, computed from the media file's +//! own bytes, and the TMDB id with season and episode for TV. Candidates come +//! back carrying everything #185 ranks on, ordered best first through +//! [`arr_core::subs::rank`] — this module reports what the API said and lets +//! the pure ranker decide what it means. +//! +//! Credentials arrive as [`OpenSubtitlesConfig`] — bootstrap config or +//! environment per §10, never a database row. An API key alone is enough to +//! search; downloading also needs a user token, which is fetched from +//! `/login` on demand and refreshed once when it expires. +//! +//! Nothing here talks to the live service in tests: every HTTP path runs +//! against fixture responses served by `wiremock`. + +use std::collections::HashMap; +use std::io::{Read, Seek, SeekFrom}; +use std::sync::Mutex; +use std::time::Duration; + +use reqwest::{Client, Method, StatusCode, Url}; +use serde::Deserialize; + +use arr_core::Language; + +use crate::{ + Candidate, CandidateId, DownloadFuture, Error, Fetched, MediaFile, MediaRef, Provider, + ProviderId, Result, SearchFuture, SearchRequest, SubtitleFormat, +}; + +/// How much of either end of the file the `moviehash` reads. The algorithm +/// OpenSubtitles.com specifies sums the file size plus both 64 KiB end chunks +/// as 8-byte little-endian words, modulo 2^64. +const HASH_CHUNK: u64 = 65_536; + +/// How much of an unexpected response body is worth keeping in an error. +const MAX_ERROR_BODY: usize = 512; + +const PROVIDER_NAME: &str = "opensubtitles"; + +const USER_AGENT: &str = "arr v0.1.0"; + +/// Compute OpenSubtitles.com's `moviehash` for a media file. +/// +/// `Ok(None)` when the file is shorter than two chunks: there are no two +/// distinct ends to read, the API rejects such files outright, so arr drops +/// the hash lane and searches by TMDB id instead. An unreadable file is a +/// real [`std::io::Error`] — searching without the winning signal silently +/// would hide exactly the kind of path problem the operator must see. +/// +/// # Errors +/// +/// Propagates [`std::io::Error`] from opening or reading the file. +pub fn moviehash(path: &std::path::Path, size: u64) -> std::io::Result> { + if size < HASH_CHUNK * 2 { + return Ok(None); + } + + let mut file = std::fs::File::open(path)?; + let mut buf = vec![0u8; HASH_CHUNK_USIZE]; + let mut hash = size; + hash = hash.wrapping_add(chunk_sum(&mut file, &mut buf)?); + file.seek(SeekFrom::Start(size - HASH_CHUNK))?; + hash = hash.wrapping_add(chunk_sum(&mut file, &mut buf)?); + Ok(Some(format!("{hash:016x}"))) +} + +const HASH_CHUNK_USIZE: usize = 65_536; + +fn chunk_sum(file: &mut std::fs::File, buf: &mut [u8]) -> std::io::Result { + file.read_exact(buf)?; + Ok(buf.as_chunks::<8>().0.iter().fold(0u64, |acc, word| { + acc.wrapping_add(u64::from_le_bytes(*word)) + })) +} + +/// Credentials for OpenSubtitles.com (DESIGN.md §10). +/// +/// The API key identifies the application; the user pair is optional but +/// required before anything can be downloaded. Never persisted to the +/// database — they reach here from bootstrap config or environment only. +#[derive(Clone, PartialEq, Eq)] +pub struct OpenSubtitlesConfig { + /// The registered API key, sent as `Api-Key` on every call. + pub api_key: String, + /// The account to download under, when configured. + pub username: Option, + /// That account's password. + pub password: Option, +} + +// Hand-written: a derived Debug would print the credentials. +impl std::fmt::Debug for OpenSubtitlesConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpenSubtitlesConfig") + .field("api_key", &"") + .field("username", &self.username.as_ref().map(|_| "")) + .field("has_password", &self.password.is_some()) + .finish() + } +} + +impl OpenSubtitlesConfig { + fn user(&self) -> Option<(&str, &str)> { + Some((self.username.as_deref()?, self.password.as_deref()?)) + } +} + +/// What search learned about one candidate that its opaque id does not +/// carry: the language it claims and the container format it downloads as. +#[derive(Clone, Debug)] +struct CandidateMeta { + language: Language, + format: SubtitleFormat, +} + +/// The OpenSubtitles.com client behind the [`Provider`] trait. +pub struct OpenSubtitles { + http: Client, + base_url: Url, + config: OpenSubtitlesConfig, + id: ProviderId, + /// The login token, fetched lazily on first download and replaced once + /// when it expires mid-flight. `None` until a download is asked for: + /// searches run with the API key alone. + token: Mutex>, + /// Facts recorded when candidates were offered, so `download` — which + /// receives only an opaque id — can still fill in `Fetched`. + meta: Mutex>, +} + +// Hand-written: the config and the token must never reach a log line. +impl std::fmt::Debug for OpenSubtitles { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpenSubtitles") + .field("base_url", &self.base_url.as_str()) + .field("id", &self.id) + .finish_non_exhaustive() + } +} + +impl OpenSubtitles { + /// A client against the real OpenSubtitles.com API. + /// + /// # Errors + /// + /// Fails if the HTTP client cannot be constructed. + pub fn new(config: OpenSubtitlesConfig) -> Result { + Self::with_base_url(config, "https://api.opensubtitles.com/api/v1/") + } + + /// Point the client somewhere else. Tests use this against `wiremock`; + /// nothing else should. + /// + /// # Errors + /// + /// Same as [`Self::new`]. + pub fn with_base_url(config: OpenSubtitlesConfig, base_url: &str) -> Result { + let id = ProviderId::new(PROVIDER_NAME); + let http = Client::builder().build().map_err(|err| Error::Transport { + provider: id.clone(), + source: Box::new(err), + })?; + let base_url = base_url.parse().map_err(|err| Error::Malformed { + provider: id.clone(), + detail: format!("bad base URL: {err}"), + })?; + Ok(Self { + http, + base_url, + config, + id, + token: Mutex::new(None), + meta: Mutex::new(HashMap::new()), + }) + } + + async fn decode Deserialize<'de>>(&self, response: reqwest::Response) -> Result { + let bytes = response.bytes().await.map_err(|err| Error::Transport { + provider: self.id.clone(), + source: Box::new(err), + })?; + serde_json::from_slice(&bytes).map_err(|err| Error::Malformed { + provider: self.id.clone(), + detail: err.to_string(), + }) + } + + /// Map every non-success status onto the error the loop can act on. + async fn check_status(&self, response: reqwest::Response) -> Result { + match response.status() { + StatusCode::OK => Ok(response), + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(Error::Unauthorized { + provider: self.id.clone(), + }), + // The documented daily-download cap answers 406; a plain rate + // limit answers 429. Both mean "come back later", not "broken". + StatusCode::TOO_MANY_REQUESTS | StatusCode::NOT_ACCEPTABLE => Err(Error::RateLimited { + provider: self.id.clone(), + retry_after: retry_after(&response), + }), + other => { + let body = response.text().await.unwrap_or_default(); + Err(Error::Malformed { + provider: self.id.clone(), + detail: format!("status {}: {}", other.as_u16(), truncate(&body)), + }) + } + } + } + + async fn send( + &self, + method: Method, + path: &str, + query: Option<&[(String, String)]>, + json: Option<&serde_json::Value>, + bearer: Option<&str>, + ) -> Result { + let url = self.base_url.join(path).map_err(|err| Error::Malformed { + provider: self.id.clone(), + detail: format!("bad request path {path}: {err}"), + })?; + + let mut request = self + .http + .request(method, url) + .header("Api-Key", &self.config.api_key) + .header("User-Agent", USER_AGENT); + if let Some(query) = query { + request = request.query(query); + } + if let Some(json) = json { + request = request.json(json); + } + if let Some(token) = bearer { + request = request.bearer_auth(token); + } + + tracing::debug!(provider = %self.id, path, "OpenSubtitles request"); + let response = request.send().await.map_err(|err| Error::Transport { + provider: self.id.clone(), + source: Box::new(err), + })?; + self.check_status(response).await + } + + /// Fetch (or refetch) the user token downloads travel under. + async fn login(&self) -> Result { + let Some((username, password)) = self.config.user() else { + return Err(Error::Unauthorized { + provider: self.id.clone(), + }); + }; + + let url = self + .base_url + .join("login") + .map_err(|err| Error::Malformed { + provider: self.id.clone(), + detail: format!("bad login path: {err}"), + })?; + let response = self + .http + .post(url) + .basic_auth(username, Some(password)) + .header("Api-Key", &self.config.api_key) + .header("User-Agent", USER_AGENT) + .send() + .await + .map_err(|err| Error::Transport { + provider: self.id.clone(), + source: Box::new(err), + })?; + let response = self.check_status(response).await?; + let body: LoginBody = self.decode(response).await?; + *self.token.lock().expect("token lock poisoned") = Some(body.token.clone()); + Ok(body.token) + } + + /// The current token, logging in first if there is none yet. + async fn current_token(&self) -> Result { + if let Some(token) = self.token.lock().expect("token lock poisoned").clone() { + return Ok(token); + } + self.login().await + } + + async fn search_inner(&self, request: &SearchRequest) -> Result> { + let hash = + moviehash(&request.file.path, request.file.size).map_err(|source| Error::Io { + path: request.file.path.clone(), + source, + })?; + + let mut params: Vec<(String, String)> = + vec![("languages".to_owned(), languages_param(&request.languages))]; + match request.file.media { + MediaRef::Movie { tmdb_id } => { + params.push(("tmdb_movie_id".to_owned(), tmdb_id.to_string())); + } + MediaRef::Episode { + tmdb_id, + season, + episode, + } => { + params.push(("tmdb_series_id".to_owned(), tmdb_id.to_string())); + params.push(("season_number".to_owned(), season.to_string())); + params.push(("episode_number".to_owned(), episode.to_string())); + } + } + if let Some(hash) = hash.as_deref() { + params.push(("moviehash".to_owned(), hash.to_owned())); + } + + let response = self + .send(Method::GET, "subtitles", Some(¶ms), None, None) + .await?; + let page: SubtitlesPage = self.decode(response).await?; + let offered: Vec<_> = page + .data + .into_iter() + .filter_map(SubtitleEntry::into_offered) + // A provider may answer with more than was asked for; ranking + // runs per wanted language, so anything else is dropped here. + .filter(|offered| request.languages.contains(&offered.candidate.language)) + .collect(); + + // Remember what each offered candidate is, so download can fill a + // Fetched in from just its id. + let mut meta = self.meta.lock().expect("meta lock poisoned"); + meta.clear(); + meta.extend(offered.iter().map(|offered| { + ( + offered.candidate.id.clone(), + CandidateMeta { + language: offered.candidate.language.clone(), + format: offered.format.clone(), + }, + ) + })); + + Ok(rank_candidates( + offered.into_iter().map(|o| o.candidate).collect(), + &request.file, + hash.as_deref(), + )) + } + + async fn download_inner(&self, id: &CandidateId) -> Result { + let Some(meta) = self + .meta + .lock() + .expect("meta lock poisoned") + .get(id) + .cloned() + else { + return Err(Error::NotFound { + provider: self.id.clone(), + candidate: id.clone(), + }); + }; + let file_id: u64 = id.as_str().parse().map_err(|_| Error::NotFound { + provider: self.id.clone(), + candidate: id.clone(), + })?; + + let token = self.current_token().await?; + let body = serde_json::json!({ "file_id": file_id }); + + let response = self + .send(Method::POST, "download", None, Some(&body), Some(&token)) + .await; + // An expired token answers 401. One refresh, then give up. + let response = match response { + Err(Error::Unauthorized { .. }) if self.config.user().is_some() => { + let fresh = self.login().await?; + self.send(Method::POST, "download", None, Some(&body), Some(&fresh)) + .await? + } + other => other?, + }; + let link: DownloadLink = self.decode(response).await?; + let content = self.fetch_bytes(&link.link).await?; + + Ok(Fetched { + id: id.clone(), + language: meta.language, + format: meta.format, + content, + }) + } + + async fn fetch_bytes(&self, link: &str) -> Result> { + let url: Url = link.parse().map_err(|err| Error::Malformed { + provider: self.id.clone(), + detail: format!("bad download link: {err}"), + })?; + let response = self + .http + .get(url) + .send() + .await + .map_err(|err| Error::Transport { + provider: self.id.clone(), + source: Box::new(err), + })?; + let response = self.check_status(response).await?; + let bytes = response.bytes().await.map_err(|err| Error::Transport { + provider: self.id.clone(), + source: Box::new(err), + })?; + Ok(bytes.to_vec()) + } +} + +impl Provider for OpenSubtitles { + fn id(&self) -> ProviderId { + self.id.clone() + } + + fn search<'a>(&'a self, request: &'a SearchRequest) -> SearchFuture<'a> { + Box::pin(async move { self.search_inner(request).await }) + } + + fn download<'a>(&'a self, id: &'a CandidateId) -> DownloadFuture<'a> { + Box::pin(async move { self.download_inner(id).await }) + } +} + +fn retry_after(response: &reqwest::Response) -> Option { + response + .headers() + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(Duration::from_secs) +} + +fn truncate(body: &str) -> String { + body.chars().take(MAX_ERROR_BODY).collect() +} + +/// One API entry turned into arr's terms, plus the facts download will need. +struct Offered { + candidate: Candidate, + format: SubtitleFormat, +} + +/// The wire shape of `POST /login`: the user token everything that touches +/// the account travels under. +#[derive(Debug, Deserialize)] +struct LoginBody { + token: String, +} + +/// The wire shape of `POST /download`: a short-lived direct link to the +/// subtitle bytes themselves. +#[derive(Debug, Deserialize)] +struct DownloadLink { + link: String, +} + +/// The wire shape of `GET /subtitles`. Only the fields ranking or the +/// download need; anything else the API adds is ignored. +#[derive(Debug, Deserialize)] +struct SubtitlesPage { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct SubtitleEntry { + attributes: SubtitleAttributes, +} + +impl SubtitleEntry { + fn into_offered(self) -> Option { + let attributes = self.attributes; + let file_id = attributes.files.first()?.file_id; + let language = language_of(attributes.language.as_deref()?)?; + let release_name = attributes.release.filter(|release| !release.is_empty()); + let format = format_of(attributes.format.as_deref()); + + Some(Offered { + candidate: Candidate { + provider: ProviderId::new(PROVIDER_NAME), + id: CandidateId::new(file_id.to_string()), + language, + hash_match: attributes.moviehash_match, + group: release_name.as_deref().and_then(group_of_release), + source: release_name.as_deref().and_then(source_of_release), + rating: attributes.ratings, + download_count: Some(attributes.download_count), + forced: attributes.foreign_parts_only, + sdh: attributes.hearing_impaired, + release_name, + }, + format, + }) + } +} + +#[derive(Debug, Deserialize)] +struct SubtitleAttributes { + language: Option, + release: Option, + #[serde(default)] + moviehash_match: bool, + ratings: Option, + #[serde(default)] + download_count: u64, + #[serde(default)] + hearing_impaired: bool, + #[serde(default)] + foreign_parts_only: bool, + format: Option, + files: Vec, +} + +#[derive(Debug, Deserialize)] +struct SubtitleFileRef { + file_id: u64, +} + +/// The API spells languages `pt-PT`, `pt-BR`, `en`; accept `_` too, since +/// tooling around this API uses it. +fn language_of(code: &str) -> Option { + match code.replace('_', "-").to_ascii_lowercase().as_str() { + "pt-pt" => Some(Language::PortuguesePortugal), + "pt-br" => Some(Language::PortugueseBrazil), + "pt" | "por" => Some(Language::PortugueseUnverified), + "" => None, + _ => Some(Language::Other(code.to_owned())), + } +} + +/// The `languages` query parameter: comma-separated codes, pt-PT and pt-BR +/// kept apart — keeping them apart is why this is the primary provider. +fn languages_param(languages: &[Language]) -> String { + languages + .iter() + .map(|language| match language { + Language::PortuguesePortugal => "pt-PT", + Language::PortugueseBrazil => "pt-BR", + Language::PortugueseUnverified => "pt", + Language::Other(tag) => tag.as_str(), + }) + .collect::>() + .join(",") +} + +/// The release group of the release a subtitle was timed against, read off +/// the release name with the same parser the indexer uses. +fn group_of_release(release: &str) -> Option { + arr_parse::parse(release).group +} + +fn source_of_release(release: &str) -> Option { + arr_parse::parse(release).source.map(Into::into) +} + +fn format_of(format: Option<&str>) -> SubtitleFormat { + match format.unwrap_or_default().to_ascii_lowercase().as_str() { + "srt" | "subrip" => SubtitleFormat::Srt, + "ass" | "ssa" => SubtitleFormat::Ass, + "vtt" | "webvtt" => SubtitleFormat::Vtt, + other => SubtitleFormat::Other(other.to_owned()), + } +} + +/// Order the candidates best first through the pure ranker (#185), building +/// the target from the same facts the search ran on: the file's own hash and +/// whatever the release name claims about group and source. +fn rank_candidates( + candidates: Vec, + file: &MediaFile, + target_hash: Option<&str>, +) -> Vec { + let claims = file.release_name.as_deref().map(arr_parse::parse); + let target = arr_core::subs::SubtitleTarget { + moviehash: target_hash, + release_name: file.release_name.as_deref(), + release_group: claims.as_ref().and_then(|claims| claims.group.as_deref()), + source: claims + .as_ref() + .and_then(|claims| claims.source.map(Into::into)), + }; + + let cores: Vec<_> = candidates.iter().map(|c| c.to_core(target_hash)).collect(); + let order: Vec = arr_core::subs::rank(&target, &cores) + .iter() + .map(|entry| entry.index) + .collect(); + + let mut slots: Vec<_> = candidates.into_iter().map(Some).collect(); + order + .iter() + .map(|&index| slots[index].take().expect("each index ranks once")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::{format_of, language_of, moviehash, rank_candidates, PROVIDER_NAME}; + use crate::{Candidate, CandidateId, MediaFile, MediaRef, ProviderId, SubtitleFormat}; + use arr_core::Language; + + #[test] + fn a_zero_filled_file_hashes_to_its_own_size() { + // Every 8-byte word of both chunks sums to zero, so the whole hash is + // the file size: 262144 = 0x40000. Hand-derived from the algorithm, + // not computed with the code under test. + let dir = tempfile::tempdir().expect("test setup"); + let path = dir.path().join("zeros.mkv"); + std::fs::write(&path, vec![0u8; 262_144]).expect("test setup"); + + let hash = moviehash(&path, 262_144).expect("test setup"); + + assert_eq!(hash.as_deref(), Some("0000000000040000")); + } + + #[test] + fn the_last_chunk_is_read_from_the_end_not_the_front() { + // First chunk all 0xFF: 8192 words of -1 sum to -8192. Last chunk all + // zero. Size 131072 - 8192 = 122880 = 0x1e000. If both ends were read + // from the front the sum would double instead. + let dir = tempfile::tempdir().expect("test setup"); + let path = dir.path().join("ends.mkv"); + let first = vec![0xFFu8; 65_536]; + let rest = vec![0u8; 65_536]; + let mut bytes = first; + bytes.extend_from_slice(&rest); + std::fs::write(&path, &bytes).expect("test setup"); + + let hash = moviehash(&path, 131_072).expect("test setup"); + + assert_eq!(hash.as_deref(), Some("000000000001e000")); + } + + #[test] + fn a_file_too_short_for_two_chunks_has_no_hash() { + let dir = tempfile::tempdir().expect("test setup"); + let path = dir.path().join("small.mkv"); + std::fs::write(&path, vec![0u8; 65_536]).expect("test setup"); + + assert_eq!(moviehash(&path, 65_536).expect("test setup"), None); + } + + #[test] + fn languages_map_both_ways_with_pt_variants_apart() { + assert_eq!(language_of("pt-PT"), Some(Language::PortuguesePortugal)); + assert_eq!(language_of("pt_BR"), Some(Language::PortugueseBrazil)); + assert_eq!(language_of("por"), Some(Language::PortugueseUnverified)); + assert_eq!(language_of("en"), Some(Language::Other("en".to_owned()))); + assert_eq!(language_of(""), None); + + assert_eq!( + super::languages_param(&[ + Language::PortuguesePortugal, + Language::PortugueseBrazil, + Language::PortugueseUnverified, + Language::Other("en".to_owned()), + ]), + "pt-PT,pt-BR,pt,en" + ); + } + + #[test] + fn formats_spell_the_way_providers_do() { + assert_eq!(format_of(Some("srt")), SubtitleFormat::Srt); + assert_eq!(format_of(Some("subrip")), SubtitleFormat::Srt); + assert_eq!(format_of(Some("ass")), SubtitleFormat::Ass); + assert_eq!(format_of(Some("vtt")), SubtitleFormat::Vtt); + assert_eq!( + format_of(Some("idx")), + SubtitleFormat::Other("idx".to_owned()) + ); + assert_eq!(format_of(None), SubtitleFormat::Other(String::new())); + } + + fn candidate(id: u64, hash_match: bool, downloads: u64) -> Candidate { + Candidate { + provider: ProviderId::new(PROVIDER_NAME), + id: CandidateId::new(id.to_string()), + language: Language::PortuguesePortugal, + hash_match, + release_name: None, + group: None, + source: None, + rating: None, + download_count: Some(downloads), + forced: false, + sdh: false, + } + } + + #[test] + fn search_results_come_back_ranked_best_first() { + let file = MediaFile { + path: "/m/f.mkv".into(), + size: 1 << 20, + release_name: None, + media: MediaRef::Movie { tmdb_id: 42 }, + }; + // The hash-matched candidate has fewer downloads; ranking puts it + // first anyway, because a moviehash match wins outright. + let candidates = vec![candidate(1, false, 9_999), candidate(2, true, 1)]; + + let ranked = rank_candidates(candidates, &file, Some("abc123")); + + let ids: Vec<_> = ranked.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(ids, ["2", "1"]); + } +} diff --git a/crates/arr-subs/tests/fixtures/subtitles_search_episode.json b/crates/arr-subs/tests/fixtures/subtitles_search_episode.json new file mode 100644 index 0000000..47ab7d8 --- /dev/null +++ b/crates/arr-subs/tests/fixtures/subtitles_search_episode.json @@ -0,0 +1,23 @@ +{ + "total_count": "1", + "data": [ + { + "id": "7111111", + "type": "subtitle", + "attributes": { + "subtitle_id": "7111111", + "language": "pt-PT", + "release": "Series.S01E02.1080p.WEB-DL-GRP", + "moviehash_match": false, + "ratings": 60.0, + "download_count": 77, + "hearing_impaired": false, + "foreign_parts_only": false, + "format": "ass", + "files": [ + { "file_id": 61111111, "file_name": "series.s01e02.pt.ass", "cd_number": 1 } + ] + } + } + ] +} diff --git a/crates/arr-subs/tests/fixtures/subtitles_search_movie.json b/crates/arr-subs/tests/fixtures/subtitles_search_movie.json new file mode 100644 index 0000000..3781b0f --- /dev/null +++ b/crates/arr-subs/tests/fixtures/subtitles_search_movie.json @@ -0,0 +1,80 @@ +{ + "total_count": "3", + "data": [ + { + "id": "7013923", + "type": "subtitle", + "attributes": { + "subtitle_id": "7013923", + "language": "pt-PT", + "release": "Dune.Part.Two.2024.1080p.WEB-DL.DDP5.1.Atmos.H.264-GRP", + "moviehash_match": true, + "ratings": 82.0, + "download_count": 12345, + "hearing_impaired": false, + "foreign_parts_only": false, + "format": "srt", + "uploader": { "name": "someone", "rating": 90 }, + "files": [ + { "file_id": 60619911, "file_name": "Dune.Part.Two.2024.1080p.WEB-DL-GRP.pt.srt", "cd_number": 1 } + ], + "feature_details": { "year": 2024, "title": "Dune: Part Two", "tmdb_id": 693134 } + } + }, + { + "id": "7000001", + "type": "subtitle", + "attributes": { + "subtitle_id": "7000001", + "language": "pt-BR", + "release": "Dune.Part.Two.2024.2160p.BluRay.GRP", + "moviehash_match": false, + "ratings": 95.0, + "download_count": 999999, + "hearing_impaired": false, + "foreign_parts_only": false, + "format": "srt", + "uploader": { "name": "other", "rating": 100 }, + "files": [ + { "file_id": 60000001, "file_name": "dune.part.two.br.srt", "cd_number": 1 } + ] + } + }, + { + "id": "6999999", + "type": "subtitle", + "attributes": { + "subtitle_id": "6999999", + "language": "en", + "release": "Dune.Part.Two.2024.1080p.WEB-DL.GRP", + "moviehash_match": true, + "ratings": 40.0, + "download_count": 5, + "hearing_impaired": true, + "foreign_parts_only": false, + "format": "srt", + "files": [ + { "file_id": 59999991, "file_name": "dune.en.sdh.srt", "cd_number": 1 } + ] + } + }, + { + "id": "6888888", + "type": "subtitle", + "attributes": { + "subtitle_id": "6888888", + "language": "fr", + "release": "Dune.Part.Two.2024.1080p.WEB-DL.GRP", + "moviehash_match": false, + "ratings": null, + "download_count": 12, + "hearing_impaired": false, + "foreign_parts_only": true, + "format": "srt", + "files": [ + { "file_id": 58888881, "file_name": "dune.fr.forced.srt", "cd_number": 1 } + ] + } + } + ] +} diff --git a/crates/arr-subs/tests/opensubtitles.rs b/crates/arr-subs/tests/opensubtitles.rs new file mode 100644 index 0000000..5f82e94 --- /dev/null +++ b/crates/arr-subs/tests/opensubtitles.rs @@ -0,0 +1,423 @@ +//! OpenSubtitles.com provider tests. Everything runs against `wiremock` — +//! the issue rules out pointing tests at the live service, which would +//! burn the daily download cap and leak a key into CI. + +// Same per-target quirk as in `lib.rs`: an integration test links the library's +// dependencies without using them directly. +use {arr_parse as _, reqwest as _, serde as _, thiserror as _, tracing as _}; + +use std::time::Duration; + +use arr_core::Language; +use arr_subs::{ + CandidateId, Error, MediaFile, MediaRef, OpenSubtitles, OpenSubtitlesConfig, Provider, + SearchRequest, SubtitleFormat, +}; +use tempfile::TempDir; +use wiremock::matchers::{body_partial_json, header, method, path, query_param}; +use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate}; + +const SEARCH_MOVIE: &str = include_str!("fixtures/subtitles_search_movie.json"); +const SEARCH_EPISODE: &str = include_str!("fixtures/subtitles_search_episode.json"); + +fn config() -> OpenSubtitlesConfig { + OpenSubtitlesConfig { + api_key: "test-api-key".to_owned(), + username: Some("user".to_owned()), + password: Some("pass".to_owned()), + } +} + +fn client(server: &MockServer) -> OpenSubtitles { + OpenSubtitles::with_base_url(config(), &format!("{}/api/v1/", server.uri())) + .expect("client builds") +} + +fn anonymous_client(server: &MockServer) -> OpenSubtitles { + OpenSubtitles::with_base_url( + OpenSubtitlesConfig { + api_key: "test-api-key".to_owned(), + username: None, + password: None, + }, + &format!("{}/api/v1/", server.uri()), + ) + .expect("client builds") +} + +/// 256 KiB of zeros hashes to its own size — see the unit tests in the +/// module — so every search against this file sends +/// `moviehash=0000000000040000`. +struct HashedFile(TempDir); + +impl HashedFile { + fn new() -> Self { + let dir = TempDir::new().expect("tempdir"); + let path = dir.path().join("dune.mkv"); + std::fs::write(&path, vec![0u8; 262_144]).expect("write"); + Self(dir) + } + + fn media(&self) -> MediaFile { + MediaFile { + path: self.0.path().join("dune.mkv"), + size: 262_144, + release_name: Some("Dune.Part.Two.2024.1080p.WEB-DL.GRP".to_owned()), + media: MediaRef::Movie { tmdb_id: 693_134 }, + } + } +} + +fn request(file: MediaFile, languages: Vec) -> SearchRequest { + SearchRequest { file, languages } +} + +async fn mount_search(server: &MockServer, body: &'static str) { + Mock::given(method("GET")) + .and(path("/api/v1/subtitles")) + .and(header("Api-Key", "test-api-key")) + .respond_with(ResponseTemplate::new(200).set_body_string(body)) + .mount(server) + .await; +} + +#[tokio::test] +async fn movie_search_sends_both_lanes_and_ranks_hash_match_first() { + let server = MockServer::start().await; + let file = HashedFile::new(); + + Mock::given(method("GET")) + .and(path("/api/v1/subtitles")) + .and(header("Api-Key", "test-api-key")) + .and(query_param("tmdb_movie_id", "693134")) + .and(query_param("moviehash", "0000000000040000")) + .and(query_param("languages", "pt-PT,pt-BR,en")) + .respond_with(ResponseTemplate::new(200).set_body_string(SEARCH_MOVIE)) + .mount(&server) + .await; + + let candidates = client(&server) + .search(&request( + file.media(), + vec![ + Language::PortuguesePortugal, + Language::PortugueseBrazil, + Language::Other("en".to_owned()), + ], + )) + .await + .expect("search succeeds"); + + // The French entry is not a wanted language and is gone; pt-PT, pt-BR + // and English remain. The hash-matched pt-PT candidate outranks the + // pt-BR one despite ten thousand times fewer downloads, because a + // moviehash match wins outright (§15). + assert_eq!(candidates.len(), 3); + assert_eq!(candidates[0].id.as_str(), "60619911"); + assert_eq!(candidates[0].language, Language::PortuguesePortugal); + assert!(candidates[0].hash_match); + + // Facts #185 ranks on survive the mapping. + assert_eq!( + candidates[0].release_name.as_deref(), + Some("Dune.Part.Two.2024.1080p.WEB-DL.DDP5.1.Atmos.H.264-GRP") + ); + assert_eq!(candidates[0].group.as_deref(), Some("GRP")); + assert_eq!(candidates[1].download_count, Some(999_999)); + // A plain subtitle outranks an SDH one for the same language — here even + // against another moviehash match, because plainness is the first tier. + assert!(candidates[2].hash_match); + assert!(candidates[2].sdh); +} + +#[tokio::test] +async fn episode_search_addresses_the_series_not_the_movie_lane() { + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/api/v1/subtitles")) + .and(query_param("tmdb_series_id", "94605")) + .and(query_param("season_number", "3")) + .and(query_param("episode_number", "7")) + .respond_with(ResponseTemplate::new(200).set_body_string(SEARCH_EPISODE)) + .mount(&server) + .await; + + let file = MediaFile { + // Below the two-chunk minimum, so the hash lane stays out and no + // real file needs to exist on disk. + path: "/m/series.mkv".into(), + size: 65_536, + release_name: None, + media: MediaRef::Episode { + tmdb_id: 94_605, + season: 3, + episode: 7, + }, + }; + let candidates = client(&server) + .search(&request(file, vec![Language::PortuguesePortugal])) + .await + .expect("search succeeds"); + + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].id.as_str(), "61111111"); +} + +/// Matches a request where the named query parameter is absent entirely. +struct QueryParamMissing(&'static str); + +impl Match for QueryParamMissing { + fn matches(&self, request: &Request) -> bool { + !request.url.query_pairs().any(|(key, _)| key == self.0) + } +} + +#[tokio::test] +async fn a_file_too_small_to_hash_searches_without_the_hash_lane() { + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/api/v1/subtitles")) + .and(query_param("tmdb_movie_id", "42")) + .and(QueryParamMissing("moviehash")) + .respond_with(ResponseTemplate::new(200).set_body_string(SEARCH_EPISODE)) + .mount(&server) + .await; + + let file = MediaFile { + path: "/m/small.mkv".into(), + size: 65_536, + release_name: None, + media: MediaRef::Movie { tmdb_id: 42 }, + }; + let candidates = client(&server) + .search(&request(file, vec![Language::PortuguesePortugal])) + .await + .expect("search succeeds"); + + assert_eq!(candidates.len(), 1); +} + +#[tokio::test] +async fn an_expired_key_is_unauthorized_and_a_cap_is_rate_limited() { + let server = MockServer::start().await; + let file = HashedFile::new(); + + Mock::given(method("GET")) + .and(path("/api/v1/subtitles")) + .and(header("Api-Key", "wrong-key")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/v1/subtitles")) + .and(header("Api-Key", "test-api-key")) + .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "3600")) + .mount(&server) + .await; + + let wrong_key = OpenSubtitles::with_base_url( + OpenSubtitlesConfig { + api_key: "wrong-key".to_owned(), + username: None, + password: None, + }, + &format!("{}/api/v1/", server.uri()), + ) + .expect("client builds"); + let err = wrong_key + .search(&request(file.media(), vec![Language::PortuguesePortugal])) + .await + .expect_err("bad key is refused"); + assert!(matches!(err, Error::Unauthorized { .. })); + + let err = client(&server) + .search(&request(file.media(), vec![Language::PortuguesePortugal])) + .await + .expect_err("cap hit"); + match err { + Error::RateLimited { retry_after, .. } => { + assert_eq!(retry_after, Some(Duration::from_hours(1))); + } + other => panic!("expected RateLimited, got {other:?}"), + } +} + +async fn mount_login(server: &MockServer, token: &'static str) { + Mock::given(method("POST")) + .and(path("/api/v1/login")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "token": token })), + ) + .mount(server) + .await; +} + +async fn mount_download_link(server: &MockServer, token: &'static str, file_id: u64) { + Mock::given(method("POST")) + .and(path("/api/v1/download")) + .and(header("Authorization", format!("Bearer {token}"))) + .and(body_partial_json(serde_json::json!({ "file_id": file_id }))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "link": format!("{}/file/{file_id}.srt", server.uri()), + "file_name": "subtitle.srt", + "requests": 99 + }))) + .mount(server) + .await; +} + +#[tokio::test] +async fn download_logs_in_then_walks_link_and_bytes_back_into_fetched() { + const SRT: &[u8] = b"1\n00:00:01,000 --> 00:00:02,000\nol\xc3\xa1\n"; + let server = MockServer::start().await; + + let file = HashedFile::new(); + mount_search(&server, SEARCH_MOVIE).await; + mount_login(&server, "token-1").await; + mount_download_link(&server, "token-1", 60_619_911).await; + + Mock::given(method("GET")) + .and(path("/file/60619911.srt")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(SRT.to_vec())) + .mount(&server) + .await; + + let provider = client(&server); + provider + .search(&request(file.media(), vec![Language::PortuguesePortugal])) + .await + .expect("search succeeds"); + + let fetched = provider + .download(&CandidateId::new("60619911")) + .await + .expect("download succeeds"); + + assert_eq!(fetched.id.as_str(), "60619911"); + assert_eq!(fetched.language, Language::PortuguesePortugal); + assert_eq!(fetched.format, SubtitleFormat::Srt); + assert_eq!(fetched.content, SRT); +} + +#[tokio::test] +async fn an_expired_token_refreshes_once_and_retries() { + let server = MockServer::start().await; + let file = HashedFile::new(); + mount_search(&server, SEARCH_MOVIE).await; + + // The first login hands out the stale token; the refresh after the 401 + // gets the fresh one. `up_to_times(1)` retires the first mock so the + // second call falls through to the next. + Mock::given(method("POST")) + .and(path("/api/v1/login")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "token": "stale" })), + ) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/api/v1/login")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "token": "fresh" })), + ) + .mount(&server) + .await; + + Mock::given(method("POST")) + .and(path("/api/v1/download")) + .and(header("Authorization", "Bearer stale")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + mount_download_link(&server, "fresh", 60_619_911).await; + Mock::given(method("GET")) + .and(path("/file/60619911.srt")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"sub".to_vec())) + .mount(&server) + .await; + + let provider = client(&server); + provider + .search(&request(file.media(), vec![Language::PortuguesePortugal])) + .await + .expect("search succeeds"); + + let fetched = provider + .download(&CandidateId::new("60619911")) + .await + .expect("the refreshed token downloads"); + + assert_eq!(fetched.content, b"sub"); +} + +#[tokio::test] +async fn hitting_the_download_cap_is_rate_limited_not_broken() { + let server = MockServer::start().await; + let file = HashedFile::new(); + mount_search(&server, SEARCH_MOVIE).await; + mount_login(&server, "t").await; + + // 406 is what the API documents for the daily-download cap. + Mock::given(method("POST")) + .and(path("/api/v1/download")) + .respond_with(ResponseTemplate::new(406).insert_header("Retry-After", "86400")) + .mount(&server) + .await; + + let provider = client(&server); + provider + .search(&request(file.media(), vec![Language::PortuguesePortugal])) + .await + .expect("search succeeds"); + + let err = provider + .download(&CandidateId::new("60619911")) + .await + .expect_err("cap reached"); + match err { + Error::RateLimited { retry_after, .. } => { + assert_eq!(retry_after, Some(Duration::from_hours(24))); + } + other => panic!("expected RateLimited, got {other:?}"), + } +} + +#[tokio::test] +async fn downloading_without_user_credentials_never_reaches_the_wire() { + let server = MockServer::start().await; + let file = HashedFile::new(); + mount_search(&server, SEARCH_MOVIE).await; + + let provider = anonymous_client(&server); + provider + .search(&request(file.media(), vec![Language::PortuguesePortugal])) + .await + .expect("search succeeds anonymously"); + + let err = provider + .download(&CandidateId::new("60619911")) + .await + .expect_err("no user configured"); + + assert!(matches!(err, Error::Unauthorized { .. })); + // No /login or /download request was ever mounted, so reaching for them + // would have failed the test with an unhandled-request error instead. +} + +#[tokio::test] +async fn an_id_this_provider_never_offered_is_not_found() { + let server = MockServer::start().await; + + let err = client(&server) + .download(&CandidateId::new("99999999")) + .await + .expect_err("unknown id"); + + match err { + Error::NotFound { candidate, .. } => assert_eq!(candidate.as_str(), "99999999"), + other => panic!("expected NotFound, got {other:?}"), + } +}