diff --git a/crates/arr-subs/src/lib.rs b/crates/arr-subs/src/lib.rs index 1b6ade9..7300e11 100644 --- a/crates/arr-subs/src/lib.rs +++ b/crates/arr-subs/src/lib.rs @@ -20,8 +20,15 @@ use std::{fmt, future::Future, pin::Pin}; +// `unused_crate_dependencies` is a per-target lint; `wiremock` is only used +// from the `tests/podnapisi.rs` integration target, not from this library +// target's own `#[cfg(test)]` module. +#[cfg(test)] +use wiremock as _; + pub mod error; pub mod model; +pub mod podnapisi; pub mod srt; pub mod translate; @@ -29,6 +36,9 @@ pub use error::{Error, Result}; pub use model::{ Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat, }; +pub use podnapisi::{ + PodnapisiProvider, PodnapisiProviderBuilder, DEFAULT_BASE_URL as PODNAPISI_DEFAULT_BASE_URL, +}; pub use srt::Cue; pub use translate::{Backend, BackendId, Batch, BatchCue, TranslateFuture, TranslatedCue}; diff --git a/crates/arr-subs/src/podnapisi.rs b/crates/arr-subs/src/podnapisi.rs new file mode 100644 index 0000000..f0e7d18 --- /dev/null +++ b/crates/arr-subs/src/podnapisi.rs @@ -0,0 +1,490 @@ +//! The Podnapisi.net provider (`DESIGN.md` §15). +//! +//! Podnapisi has no `moviehash` lane: it matches on a text search over the +//! release name it was imported under, nothing else. Every candidate this +//! provider returns therefore carries `hash_match: false`, which is exactly +//! what ranking (#185) already expresses without a special case — a plain +//! release-name match and below. +//! +//! The search API is unauthenticated JSON, and download hands back a zip +//! containing exactly one subtitle file. Both are undocumented officially; +//! this follows the shape `subliminal`'s maintained provider uses against the +//! live service. + +use std::{ + collections::HashSet, + io::{Cursor, Read as _}, + time::Duration, +}; + +use arr_core::{ + lang::{resolve_portuguese, PortugueseEvidence}, + Language, +}; +use reqwest::{StatusCode, Url}; +use serde::Deserialize; + +use crate::{ + error::{Error, Result}, + model::{ + Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, + SubtitleFormat, + }, + DownloadFuture, Provider, SearchFuture, +}; + +/// Podnapisi's public search and download root. +pub const DEFAULT_BASE_URL: &str = "https://www.podnapisi.net/subtitles/"; + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); + +/// A results page claiming more pages than this is treated as malformed +/// rather than looped over forever. +const MAX_PAGES: u32 = 50; + +fn provider_id() -> ProviderId { + ProviderId::new("podnapisi") +} + +/// The Podnapisi.net subtitle provider. +#[derive(Debug)] +pub struct PodnapisiProvider { + http: reqwest::Client, + base_url: Url, +} + +impl PodnapisiProvider { + /// A provider against the real Podnapisi.net, with default timeouts. + /// + /// # Errors + /// + /// [`Error::Config`] if the HTTP client cannot be built. + pub fn new() -> Result { + Self::builder().build() + } + + /// Start configuring a provider. + #[must_use] + pub fn builder() -> PodnapisiProviderBuilder { + PodnapisiProviderBuilder { + base_url: DEFAULT_BASE_URL.to_owned(), + timeout: DEFAULT_TIMEOUT, + } + } + + async fn search_language( + &self, + file: &MediaFile, + language: &Language, + ) -> Result> { + let Some(keywords) = file.release_name.as_deref() else { + // No release name, nothing to search Podnapisi's keyword index + // with. Not an error: the provider simply has nothing to offer. + return Ok(Vec::new()); + }; + + let mut candidates = Vec::new(); + let mut seen_pids = HashSet::new(); + let mut page = 1; + + loop { + let response = self + .fetch_page(keywords, language, &file.media, page) + .await?; + let all_pages = response.all_pages.max(1); + + for item in response.data { + if !seen_pids.insert(item.id.clone()) { + continue; + } + candidates.push(item.into_candidate(language)); + } + + if response.page >= all_pages || page >= MAX_PAGES { + break; + } + page += 1; + } + + Ok(candidates) + } + + async fn fetch_page( + &self, + keywords: &str, + language: &Language, + media: &MediaRef, + page: u32, + ) -> Result { + let mut url = self + .base_url + .join("search/advanced") + .map_err(|err| config_error(format!("invalid search URL: {err}")))?; + + { + let mut params = url.query_pairs_mut(); + params.append_pair("keywords", keywords); + params.append_pair("language", &language_param(language)); + match media { + MediaRef::Movie { .. } => { + params.append_pair("movie_type", "movie"); + } + MediaRef::Episode { + season, episode, .. + } => { + params.append_pair("movie_type", "tv-series"); + params.append_pair("movie_type", "mini-series"); + params.append_pair("seasons", &season.to_string()); + params.append_pair("episodes", &episode.to_string()); + } + } + if page > 1 { + params.append_pair("page", &page.to_string()); + } + } + + let body = self.get(url).await?; + serde_json::from_str(&body).map_err(|err| Error::Malformed { + provider: provider_id(), + detail: format!("search response did not parse: {err}"), + }) + } + + /// Download one candidate this provider offered. + async fn download_candidate(&self, id: &CandidateId) -> Result { + let (pid, language) = split_candidate_id(id)?; + + let mut url = self + .base_url + .join(&format!("{pid}/download")) + .map_err(|err| config_error(format!("invalid download URL: {err}")))?; + url.query_pairs_mut().append_pair("container", "zip"); + + let bytes = self.get_bytes(url, id).await?; + let (content, format) = extract_single_file(&bytes)?; + + Ok(Fetched { + id: id.clone(), + language, + format, + content, + }) + } + + async fn get(&self, url: Url) -> Result { + let response = self.send(url, None).await?; + response.text().await.map_err(|err| Error::Transport { + provider: provider_id(), + source: Box::new(err), + }) + } + + async fn get_bytes(&self, url: Url, candidate: &CandidateId) -> Result> { + let response = self.send(url, Some(candidate)).await?; + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|err| Error::Transport { + provider: provider_id(), + source: Box::new(err), + }) + } + + /// `candidate` is only known when the request is a download — it names + /// the [`Error::NotFound`] a 404 becomes. + async fn send(&self, url: Url, candidate: Option<&CandidateId>) -> Result { + tracing::debug!(url = %url, "Podnapisi request"); + let response = self + .http + .get(url) + .send() + .await + .map_err(|err| Error::Transport { + provider: provider_id(), + source: Box::new(err), + })?; + + let status = response.status(); + if status.is_success() { + return Ok(response); + } + + Err(match (status, candidate) { + (StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN, _) => Error::Unauthorized { + provider: provider_id(), + }, + (StatusCode::TOO_MANY_REQUESTS, _) => Error::RateLimited { + provider: provider_id(), + retry_after: retry_after(&response), + }, + (StatusCode::NOT_FOUND, Some(candidate)) => Error::NotFound { + provider: provider_id(), + candidate: candidate.clone(), + }, + (other, _) => Error::Malformed { + provider: provider_id(), + detail: format!("HTTP {other}"), + }, + }) + } +} + +fn config_error(detail: String) -> Error { + Error::Config { + provider: provider_id(), + detail, + } +} + +impl Provider for PodnapisiProvider { + fn id(&self) -> ProviderId { + provider_id() + } + + fn search<'a>(&'a self, request: &'a SearchRequest) -> SearchFuture<'a> { + Box::pin(async move { + let mut candidates = Vec::new(); + for language in &request.languages { + candidates.extend(self.search_language(&request.file, language).await?); + } + Ok(candidates) + }) + } + + fn download<'a>(&'a self, id: &'a CandidateId) -> DownloadFuture<'a> { + Box::pin(async move { self.download_candidate(id).await }) + } +} + +/// Configuration for a [`PodnapisiProvider`]. +#[derive(Debug, Clone)] +pub struct PodnapisiProviderBuilder { + base_url: String, + timeout: Duration, +} + +impl PodnapisiProviderBuilder { + /// Point the provider somewhere other than Podnapisi.net. Tests use + /// this; nothing else should. + #[must_use] + pub fn base_url(mut self, base_url: impl Into) -> Self { + self.base_url = base_url.into(); + self + } + + /// Per-request timeout. + #[must_use] + pub fn timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + /// Build the provider. + /// + /// # Errors + /// + /// [`Error::Config`] if the base URL will not parse or the HTTP client + /// cannot be built. + pub fn build(self) -> Result { + let mut base_url = self.base_url; + if !base_url.ends_with('/') { + base_url.push('/'); + } + let base_url = Url::parse(&base_url).map_err(|err| Error::Config { + provider: provider_id(), + detail: format!("invalid base URL: {err}"), + })?; + + let http = reqwest::Client::builder() + .timeout(self.timeout) + .user_agent(concat!("arr/", env!("CARGO_PKG_VERSION"))) + .build() + .map_err(|err| Error::Config { + provider: provider_id(), + detail: err.to_string(), + })?; + + Ok(PodnapisiProvider { http, base_url }) + } +} + +/// One page of Podnapisi's `search/advanced` response. +#[derive(Debug, Deserialize)] +struct SearchResponse { + data: Vec, + page: u32, + all_pages: u32, +} + +#[derive(Debug, Deserialize)] +struct RawCandidate { + id: String, + language: String, + #[serde(default)] + flags: Vec, + #[serde(default)] + releases: Vec, + #[serde(default)] + custom_releases: Vec, +} + +impl RawCandidate { + fn into_candidate(self, requested: &Language) -> Candidate { + let release_name = self.releases.into_iter().chain(self.custom_releases).next(); + let claims = release_name.as_deref().map(arr_parse::parse); + + let language = resolve_language( + &self.language, + claims.as_ref().map_or(&[], |claims| &claims.languages[..]), + requested, + ); + + Candidate { + provider: provider_id(), + id: CandidateId::new(format!("{}|{language}", self.id)), + language, + hash_match: false, + group: claims.as_ref().and_then(|claims| claims.group.clone()), + source: claims.and_then(|claims| claims.source).map(Into::into), + release_name, + rating: None, + download_count: None, + forced: false, + sdh: self.flags.iter().any(|flag| flag == "hearing_impaired"), + } + } +} + +/// Turn a wanted [`Language`] into the string Podnapisi's `language` query +/// parameter expects — the same BCP-47-shaped spelling [`Language::Display`] +/// already produces for the pt-PT/pt-BR split, lower-cased where this crate +/// has no opinion. +fn language_param(language: &Language) -> String { + match language { + Language::PortuguesePortugal => "pt-PT".to_owned(), + Language::PortugueseBrazil => "pt-BR".to_owned(), + Language::PortugueseUnverified => "pt".to_owned(), + Language::Other(tag) => tag.clone(), + } +} + +/// Resolve Podnapisi's own `language` tag on a result, using the release +/// name's claimed language markers as the first signal — the same signal +/// order `arr-core::lang` documents, with the container tag Podnapisi sends +/// standing in for a BCP-47 tag. +fn resolve_language( + raw_tag: &str, + name_markers: &[arr_parse::LanguageMarker], + requested: &Language, +) -> Language { + let primary = raw_tag + .split(['-', '_']) + .next() + .unwrap_or(raw_tag) + .to_ascii_lowercase(); + + if primary == "pt" || primary == "por" { + return resolve_portuguese(PortugueseEvidence { + name_markers, + stream_title: None, + handler_name: None, + container_tag: Some(raw_tag), + }); + } + + // Podnapisi echoes back whatever it matched the search on; when the tag + // carries no useful primary subtag, the language we asked for is a + // better answer than a guess. + if primary.is_empty() { + return requested.clone(); + } + + Language::Other(primary) +} + +/// The `CandidateId` this provider issues is `{pid}|{language}`, the +/// language spelled exactly as [`Language::Display`] renders it — so +/// `download` recovers it without a second round trip to disambiguate +/// pt-PT from pt-BR. +fn split_candidate_id(id: &CandidateId) -> Result<(&str, Language)> { + id.as_str() + .rsplit_once('|') + .map(|(pid, language)| (pid, parse_language_tag(language))) + .ok_or_else(|| Error::NotFound { + provider: provider_id(), + candidate: id.clone(), + }) +} + +/// The inverse of [`Language::Display`], for the tag this module itself +/// wrote into a [`CandidateId`]. Not a general parser: it only has to +/// round-trip the four spellings `Display` can produce. +fn parse_language_tag(tag: &str) -> Language { + match tag { + "pt-PT" => Language::PortuguesePortugal, + "pt-BR" => Language::PortugueseBrazil, + "por-unverified" => Language::PortugueseUnverified, + other => Language::Other(other.to_owned()), + } +} + +fn retry_after(response: &reqwest::Response) -> Option { + response + .headers() + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(Duration::from_secs) +} + +/// Unzip a Podnapisi download, which is always a single subtitle file. +/// +/// # Errors +/// +/// [`Error::Malformed`] if the body is not a zip, or does not contain +/// exactly one file. +fn extract_single_file(bytes: &[u8]) -> Result<(Vec, SubtitleFormat)> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|err| Error::Malformed { + provider: provider_id(), + detail: format!("download was not a zip: {err}"), + })?; + + if archive.len() != 1 { + return Err(Error::Malformed { + provider: provider_id(), + detail: format!("zip contained {} files, expected 1", archive.len()), + }); + } + + let mut entry = archive.by_index(0).map_err(|err| Error::Malformed { + provider: provider_id(), + detail: format!("could not read the zipped subtitle: {err}"), + })?; + + let format = + entry + .name() + .rsplit_once('.') + .map_or(SubtitleFormat::Other(String::new()), |(_, ext)| { + match ext.to_ascii_lowercase().as_str() { + "srt" => SubtitleFormat::Srt, + "ass" | "ssa" => SubtitleFormat::Ass, + "vtt" => SubtitleFormat::Vtt, + other => SubtitleFormat::Other(other.to_owned()), + } + }); + + let mut content = Vec::new(); + entry + .read_to_end(&mut content) + .map_err(|err| Error::Malformed { + provider: provider_id(), + detail: format!("could not read the zipped subtitle: {err}"), + })?; + + Ok((content, format)) +}