feat(arr): add opensubtitles.com subtitle provider
Searches by moviehash computed from the media file and by TMDB id with season/episode for TV, ranks candidates through arr_core::subs::rank, and downloads under a lazily-fetched user token. The daily download cap (429/406) surfaces as Error::RateLimited so the loop can show a queue state. Credentials come from config or environment only; tests run against wiremock fixtures.
This commit is contained in:
Generated
+7
@@ -205,8 +205,15 @@ name = "arr-subs"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arr-core",
|
"arr-core",
|
||||||
|
"arr-parse",
|
||||||
|
"reqwest",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
"wiremock",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
@@ -19,10 +19,17 @@ translate-command = []
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
arr-core.workspace = true
|
arr-core.workspace = true
|
||||||
|
arr-parse.workspace = true
|
||||||
|
reqwest.workspace = true
|
||||||
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
|
tracing.workspace = true
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
tempfile.workspace = true
|
||||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||||
|
wiremock.workspace = true
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ use std::{fmt, future::Future, pin::Pin};
|
|||||||
|
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
|
pub mod opensubtitles;
|
||||||
pub mod srt;
|
pub mod srt;
|
||||||
pub mod translate;
|
pub mod translate;
|
||||||
|
|
||||||
@@ -29,9 +30,16 @@ pub use error::{Error, Result};
|
|||||||
pub use model::{
|
pub use model::{
|
||||||
Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat,
|
Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat,
|
||||||
};
|
};
|
||||||
|
pub use opensubtitles::{moviehash, OpenSubtitles, OpenSubtitlesConfig};
|
||||||
pub use srt::Cue;
|
pub use srt::Cue;
|
||||||
pub use translate::{Backend, BackendId, Batch, BatchCue, TranslateFuture, TranslatedCue};
|
pub use translate::{Backend, BackendId, Batch, BatchCue, TranslateFuture, TranslatedCue};
|
||||||
|
|
||||||
|
// The OpenSubtitles provider's HTTP paths are exercised by the integration
|
||||||
|
// tests, which link this crate; the unit-test target still has to satisfy
|
||||||
|
// the unused-crate-dependencies lint for the dev-dependency itself.
|
||||||
|
#[cfg(test)]
|
||||||
|
use wiremock as _;
|
||||||
|
|
||||||
/// The candidates one search turned up.
|
/// The candidates one search turned up.
|
||||||
pub type SearchFuture<'a> = Pin<Box<dyn Future<Output = Result<Vec<Candidate>>> + Send + 'a>>;
|
pub type SearchFuture<'a> = Pin<Box<dyn Future<Output = Result<Vec<Candidate>>> + Send + 'a>>;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,723 @@
|
|||||||
|
//! The OpenSubtitles.com provider (`DESIGN.md` §15, issue #187).
|
||||||
|
//!
|
||||||
|
//! One of the two providers §15 names, and the primary one: it has an
|
||||||
|
//! official REST API, it keeps pt-PT and pt-BR apart properly, and it is the
|
||||||
|
//! only provider that answers `moviehash` searches — the signal that wins
|
||||||
|
//! ranking outright.
|
||||||
|
//!
|
||||||
|
//! Both lanes feed one search: `moviehash`, computed from the media file's
|
||||||
|
//! own bytes, and the TMDB id with season and episode for TV. Candidates come
|
||||||
|
//! back carrying everything #185 ranks on, ordered best first through
|
||||||
|
//! [`arr_core::subs::rank`] — this module reports what the API said and lets
|
||||||
|
//! the pure ranker decide what it means.
|
||||||
|
//!
|
||||||
|
//! Credentials arrive as [`OpenSubtitlesConfig`] — bootstrap config or
|
||||||
|
//! environment per §10, never a database row. An API key alone is enough to
|
||||||
|
//! search; downloading also needs a user token, which is fetched from
|
||||||
|
//! `/login` on demand and refreshed once when it expires.
|
||||||
|
//!
|
||||||
|
//! Nothing here talks to the live service in tests: every HTTP path runs
|
||||||
|
//! against fixture responses served by `wiremock`.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::io::{Read, Seek, SeekFrom};
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use reqwest::{Client, Method, StatusCode, Url};
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
use arr_core::Language;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
Candidate, CandidateId, DownloadFuture, Error, Fetched, MediaFile, MediaRef, Provider,
|
||||||
|
ProviderId, Result, SearchFuture, SearchRequest, SubtitleFormat,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// How much of either end of the file the `moviehash` reads. The algorithm
|
||||||
|
/// OpenSubtitles.com specifies sums the file size plus both 64 KiB end chunks
|
||||||
|
/// as 8-byte little-endian words, modulo 2^64.
|
||||||
|
const HASH_CHUNK: u64 = 65_536;
|
||||||
|
|
||||||
|
/// How much of an unexpected response body is worth keeping in an error.
|
||||||
|
const MAX_ERROR_BODY: usize = 512;
|
||||||
|
|
||||||
|
const PROVIDER_NAME: &str = "opensubtitles";
|
||||||
|
|
||||||
|
const USER_AGENT: &str = "arr v0.1.0";
|
||||||
|
|
||||||
|
/// Compute OpenSubtitles.com's `moviehash` for a media file.
|
||||||
|
///
|
||||||
|
/// `Ok(None)` when the file is shorter than two chunks: there are no two
|
||||||
|
/// distinct ends to read, the API rejects such files outright, so arr drops
|
||||||
|
/// the hash lane and searches by TMDB id instead. An unreadable file is a
|
||||||
|
/// real [`std::io::Error`] — searching without the winning signal silently
|
||||||
|
/// would hide exactly the kind of path problem the operator must see.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Propagates [`std::io::Error`] from opening or reading the file.
|
||||||
|
pub fn moviehash(path: &std::path::Path, size: u64) -> std::io::Result<Option<String>> {
|
||||||
|
if size < HASH_CHUNK * 2 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = std::fs::File::open(path)?;
|
||||||
|
let mut buf = vec![0u8; HASH_CHUNK_USIZE];
|
||||||
|
let mut hash = size;
|
||||||
|
hash = hash.wrapping_add(chunk_sum(&mut file, &mut buf)?);
|
||||||
|
file.seek(SeekFrom::Start(size - HASH_CHUNK))?;
|
||||||
|
hash = hash.wrapping_add(chunk_sum(&mut file, &mut buf)?);
|
||||||
|
Ok(Some(format!("{hash:016x}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
const HASH_CHUNK_USIZE: usize = 65_536;
|
||||||
|
|
||||||
|
fn chunk_sum(file: &mut std::fs::File, buf: &mut [u8]) -> std::io::Result<u64> {
|
||||||
|
file.read_exact(buf)?;
|
||||||
|
Ok(buf.as_chunks::<8>().0.iter().fold(0u64, |acc, word| {
|
||||||
|
acc.wrapping_add(u64::from_le_bytes(*word))
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Credentials for OpenSubtitles.com (DESIGN.md §10).
|
||||||
|
///
|
||||||
|
/// The API key identifies the application; the user pair is optional but
|
||||||
|
/// required before anything can be downloaded. Never persisted to the
|
||||||
|
/// database — they reach here from bootstrap config or environment only.
|
||||||
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
|
pub struct OpenSubtitlesConfig {
|
||||||
|
/// The registered API key, sent as `Api-Key` on every call.
|
||||||
|
pub api_key: String,
|
||||||
|
/// The account to download under, when configured.
|
||||||
|
pub username: Option<String>,
|
||||||
|
/// That account's password.
|
||||||
|
pub password: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hand-written: a derived Debug would print the credentials.
|
||||||
|
impl std::fmt::Debug for OpenSubtitlesConfig {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("OpenSubtitlesConfig")
|
||||||
|
.field("api_key", &"<redacted>")
|
||||||
|
.field("username", &self.username.as_ref().map(|_| "<redacted>"))
|
||||||
|
.field("has_password", &self.password.is_some())
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OpenSubtitlesConfig {
|
||||||
|
fn user(&self) -> Option<(&str, &str)> {
|
||||||
|
Some((self.username.as_deref()?, self.password.as_deref()?))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What search learned about one candidate that its opaque id does not
|
||||||
|
/// carry: the language it claims and the container format it downloads as.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct CandidateMeta {
|
||||||
|
language: Language,
|
||||||
|
format: SubtitleFormat,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The OpenSubtitles.com client behind the [`Provider`] trait.
|
||||||
|
pub struct OpenSubtitles {
|
||||||
|
http: Client,
|
||||||
|
base_url: Url,
|
||||||
|
config: OpenSubtitlesConfig,
|
||||||
|
id: ProviderId,
|
||||||
|
/// The login token, fetched lazily on first download and replaced once
|
||||||
|
/// when it expires mid-flight. `None` until a download is asked for:
|
||||||
|
/// searches run with the API key alone.
|
||||||
|
token: Mutex<Option<String>>,
|
||||||
|
/// Facts recorded when candidates were offered, so `download` — which
|
||||||
|
/// receives only an opaque id — can still fill in `Fetched`.
|
||||||
|
meta: Mutex<HashMap<CandidateId, CandidateMeta>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hand-written: the config and the token must never reach a log line.
|
||||||
|
impl std::fmt::Debug for OpenSubtitles {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("OpenSubtitles")
|
||||||
|
.field("base_url", &self.base_url.as_str())
|
||||||
|
.field("id", &self.id)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OpenSubtitles {
|
||||||
|
/// A client against the real OpenSubtitles.com API.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Fails if the HTTP client cannot be constructed.
|
||||||
|
pub fn new(config: OpenSubtitlesConfig) -> Result<Self> {
|
||||||
|
Self::with_base_url(config, "https://api.opensubtitles.com/api/v1/")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Point the client somewhere else. Tests use this against `wiremock`;
|
||||||
|
/// nothing else should.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Same as [`Self::new`].
|
||||||
|
pub fn with_base_url(config: OpenSubtitlesConfig, base_url: &str) -> Result<Self> {
|
||||||
|
let id = ProviderId::new(PROVIDER_NAME);
|
||||||
|
let http = Client::builder().build().map_err(|err| Error::Transport {
|
||||||
|
provider: id.clone(),
|
||||||
|
source: Box::new(err),
|
||||||
|
})?;
|
||||||
|
let base_url = base_url.parse().map_err(|err| Error::Malformed {
|
||||||
|
provider: id.clone(),
|
||||||
|
detail: format!("bad base URL: {err}"),
|
||||||
|
})?;
|
||||||
|
Ok(Self {
|
||||||
|
http,
|
||||||
|
base_url,
|
||||||
|
config,
|
||||||
|
id,
|
||||||
|
token: Mutex::new(None),
|
||||||
|
meta: Mutex::new(HashMap::new()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn decode<T: for<'de> Deserialize<'de>>(&self, response: reqwest::Response) -> Result<T> {
|
||||||
|
let bytes = response.bytes().await.map_err(|err| Error::Transport {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
source: Box::new(err),
|
||||||
|
})?;
|
||||||
|
serde_json::from_slice(&bytes).map_err(|err| Error::Malformed {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
detail: err.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map every non-success status onto the error the loop can act on.
|
||||||
|
async fn check_status(&self, response: reqwest::Response) -> Result<reqwest::Response> {
|
||||||
|
match response.status() {
|
||||||
|
StatusCode::OK => Ok(response),
|
||||||
|
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(Error::Unauthorized {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
}),
|
||||||
|
// The documented daily-download cap answers 406; a plain rate
|
||||||
|
// limit answers 429. Both mean "come back later", not "broken".
|
||||||
|
StatusCode::TOO_MANY_REQUESTS | StatusCode::NOT_ACCEPTABLE => Err(Error::RateLimited {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
retry_after: retry_after(&response),
|
||||||
|
}),
|
||||||
|
other => {
|
||||||
|
let body = response.text().await.unwrap_or_default();
|
||||||
|
Err(Error::Malformed {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
detail: format!("status {}: {}", other.as_u16(), truncate(&body)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn send(
|
||||||
|
&self,
|
||||||
|
method: Method,
|
||||||
|
path: &str,
|
||||||
|
query: Option<&[(String, String)]>,
|
||||||
|
json: Option<&serde_json::Value>,
|
||||||
|
bearer: Option<&str>,
|
||||||
|
) -> Result<reqwest::Response> {
|
||||||
|
let url = self.base_url.join(path).map_err(|err| Error::Malformed {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
detail: format!("bad request path {path}: {err}"),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut request = self
|
||||||
|
.http
|
||||||
|
.request(method, url)
|
||||||
|
.header("Api-Key", &self.config.api_key)
|
||||||
|
.header("User-Agent", USER_AGENT);
|
||||||
|
if let Some(query) = query {
|
||||||
|
request = request.query(query);
|
||||||
|
}
|
||||||
|
if let Some(json) = json {
|
||||||
|
request = request.json(json);
|
||||||
|
}
|
||||||
|
if let Some(token) = bearer {
|
||||||
|
request = request.bearer_auth(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::debug!(provider = %self.id, path, "OpenSubtitles request");
|
||||||
|
let response = request.send().await.map_err(|err| Error::Transport {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
source: Box::new(err),
|
||||||
|
})?;
|
||||||
|
self.check_status(response).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch (or refetch) the user token downloads travel under.
|
||||||
|
async fn login(&self) -> Result<String> {
|
||||||
|
let Some((username, password)) = self.config.user() else {
|
||||||
|
return Err(Error::Unauthorized {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
let url = self
|
||||||
|
.base_url
|
||||||
|
.join("login")
|
||||||
|
.map_err(|err| Error::Malformed {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
detail: format!("bad login path: {err}"),
|
||||||
|
})?;
|
||||||
|
let response = self
|
||||||
|
.http
|
||||||
|
.post(url)
|
||||||
|
.basic_auth(username, Some(password))
|
||||||
|
.header("Api-Key", &self.config.api_key)
|
||||||
|
.header("User-Agent", USER_AGENT)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| Error::Transport {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
source: Box::new(err),
|
||||||
|
})?;
|
||||||
|
let response = self.check_status(response).await?;
|
||||||
|
let body: LoginBody = self.decode(response).await?;
|
||||||
|
*self.token.lock().expect("token lock poisoned") = Some(body.token.clone());
|
||||||
|
Ok(body.token)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The current token, logging in first if there is none yet.
|
||||||
|
async fn current_token(&self) -> Result<String> {
|
||||||
|
if let Some(token) = self.token.lock().expect("token lock poisoned").clone() {
|
||||||
|
return Ok(token);
|
||||||
|
}
|
||||||
|
self.login().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn search_inner(&self, request: &SearchRequest) -> Result<Vec<Candidate>> {
|
||||||
|
let hash =
|
||||||
|
moviehash(&request.file.path, request.file.size).map_err(|source| Error::Io {
|
||||||
|
path: request.file.path.clone(),
|
||||||
|
source,
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut params: Vec<(String, String)> =
|
||||||
|
vec![("languages".to_owned(), languages_param(&request.languages))];
|
||||||
|
match request.file.media {
|
||||||
|
MediaRef::Movie { tmdb_id } => {
|
||||||
|
params.push(("tmdb_movie_id".to_owned(), tmdb_id.to_string()));
|
||||||
|
}
|
||||||
|
MediaRef::Episode {
|
||||||
|
tmdb_id,
|
||||||
|
season,
|
||||||
|
episode,
|
||||||
|
} => {
|
||||||
|
params.push(("tmdb_series_id".to_owned(), tmdb_id.to_string()));
|
||||||
|
params.push(("season_number".to_owned(), season.to_string()));
|
||||||
|
params.push(("episode_number".to_owned(), episode.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(hash) = hash.as_deref() {
|
||||||
|
params.push(("moviehash".to_owned(), hash.to_owned()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = self
|
||||||
|
.send(Method::GET, "subtitles", Some(¶ms), None, None)
|
||||||
|
.await?;
|
||||||
|
let page: SubtitlesPage = self.decode(response).await?;
|
||||||
|
let offered: Vec<_> = page
|
||||||
|
.data
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(SubtitleEntry::into_offered)
|
||||||
|
// A provider may answer with more than was asked for; ranking
|
||||||
|
// runs per wanted language, so anything else is dropped here.
|
||||||
|
.filter(|offered| request.languages.contains(&offered.candidate.language))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Remember what each offered candidate is, so download can fill a
|
||||||
|
// Fetched in from just its id.
|
||||||
|
let mut meta = self.meta.lock().expect("meta lock poisoned");
|
||||||
|
meta.clear();
|
||||||
|
meta.extend(offered.iter().map(|offered| {
|
||||||
|
(
|
||||||
|
offered.candidate.id.clone(),
|
||||||
|
CandidateMeta {
|
||||||
|
language: offered.candidate.language.clone(),
|
||||||
|
format: offered.format.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
|
||||||
|
Ok(rank_candidates(
|
||||||
|
offered.into_iter().map(|o| o.candidate).collect(),
|
||||||
|
&request.file,
|
||||||
|
hash.as_deref(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn download_inner(&self, id: &CandidateId) -> Result<Fetched> {
|
||||||
|
let Some(meta) = self
|
||||||
|
.meta
|
||||||
|
.lock()
|
||||||
|
.expect("meta lock poisoned")
|
||||||
|
.get(id)
|
||||||
|
.cloned()
|
||||||
|
else {
|
||||||
|
return Err(Error::NotFound {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
candidate: id.clone(),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
let file_id: u64 = id.as_str().parse().map_err(|_| Error::NotFound {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
candidate: id.clone(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let token = self.current_token().await?;
|
||||||
|
let body = serde_json::json!({ "file_id": file_id });
|
||||||
|
|
||||||
|
let response = self
|
||||||
|
.send(Method::POST, "download", None, Some(&body), Some(&token))
|
||||||
|
.await;
|
||||||
|
// An expired token answers 401. One refresh, then give up.
|
||||||
|
let response = match response {
|
||||||
|
Err(Error::Unauthorized { .. }) if self.config.user().is_some() => {
|
||||||
|
let fresh = self.login().await?;
|
||||||
|
self.send(Method::POST, "download", None, Some(&body), Some(&fresh))
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
other => other?,
|
||||||
|
};
|
||||||
|
let link: DownloadLink = self.decode(response).await?;
|
||||||
|
let content = self.fetch_bytes(&link.link).await?;
|
||||||
|
|
||||||
|
Ok(Fetched {
|
||||||
|
id: id.clone(),
|
||||||
|
language: meta.language,
|
||||||
|
format: meta.format,
|
||||||
|
content,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_bytes(&self, link: &str) -> Result<Vec<u8>> {
|
||||||
|
let url: Url = link.parse().map_err(|err| Error::Malformed {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
detail: format!("bad download link: {err}"),
|
||||||
|
})?;
|
||||||
|
let response = self
|
||||||
|
.http
|
||||||
|
.get(url)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| Error::Transport {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
source: Box::new(err),
|
||||||
|
})?;
|
||||||
|
let response = self.check_status(response).await?;
|
||||||
|
let bytes = response.bytes().await.map_err(|err| Error::Transport {
|
||||||
|
provider: self.id.clone(),
|
||||||
|
source: Box::new(err),
|
||||||
|
})?;
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Provider for OpenSubtitles {
|
||||||
|
fn id(&self) -> ProviderId {
|
||||||
|
self.id.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn search<'a>(&'a self, request: &'a SearchRequest) -> SearchFuture<'a> {
|
||||||
|
Box::pin(async move { self.search_inner(request).await })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn download<'a>(&'a self, id: &'a CandidateId) -> DownloadFuture<'a> {
|
||||||
|
Box::pin(async move { self.download_inner(id).await })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retry_after(response: &reqwest::Response) -> Option<Duration> {
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get(reqwest::header::RETRY_AFTER)?
|
||||||
|
.to_str()
|
||||||
|
.ok()?
|
||||||
|
.trim()
|
||||||
|
.parse::<u64>()
|
||||||
|
.ok()
|
||||||
|
.map(Duration::from_secs)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate(body: &str) -> String {
|
||||||
|
body.chars().take(MAX_ERROR_BODY).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One API entry turned into arr's terms, plus the facts download will need.
|
||||||
|
struct Offered {
|
||||||
|
candidate: Candidate,
|
||||||
|
format: SubtitleFormat,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The wire shape of `POST /login`: the user token everything that touches
|
||||||
|
/// the account travels under.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct LoginBody {
|
||||||
|
token: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The wire shape of `POST /download`: a short-lived direct link to the
|
||||||
|
/// subtitle bytes themselves.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct DownloadLink {
|
||||||
|
link: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The wire shape of `GET /subtitles`. Only the fields ranking or the
|
||||||
|
/// download need; anything else the API adds is ignored.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SubtitlesPage {
|
||||||
|
data: Vec<SubtitleEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SubtitleEntry {
|
||||||
|
attributes: SubtitleAttributes,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SubtitleEntry {
|
||||||
|
fn into_offered(self) -> Option<Offered> {
|
||||||
|
let attributes = self.attributes;
|
||||||
|
let file_id = attributes.files.first()?.file_id;
|
||||||
|
let language = language_of(attributes.language.as_deref()?)?;
|
||||||
|
let release_name = attributes.release.filter(|release| !release.is_empty());
|
||||||
|
let format = format_of(attributes.format.as_deref());
|
||||||
|
|
||||||
|
Some(Offered {
|
||||||
|
candidate: Candidate {
|
||||||
|
provider: ProviderId::new(PROVIDER_NAME),
|
||||||
|
id: CandidateId::new(file_id.to_string()),
|
||||||
|
language,
|
||||||
|
hash_match: attributes.moviehash_match,
|
||||||
|
group: release_name.as_deref().and_then(group_of_release),
|
||||||
|
source: release_name.as_deref().and_then(source_of_release),
|
||||||
|
rating: attributes.ratings,
|
||||||
|
download_count: Some(attributes.download_count),
|
||||||
|
forced: attributes.foreign_parts_only,
|
||||||
|
sdh: attributes.hearing_impaired,
|
||||||
|
release_name,
|
||||||
|
},
|
||||||
|
format,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SubtitleAttributes {
|
||||||
|
language: Option<String>,
|
||||||
|
release: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
moviehash_match: bool,
|
||||||
|
ratings: Option<f32>,
|
||||||
|
#[serde(default)]
|
||||||
|
download_count: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
hearing_impaired: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
foreign_parts_only: bool,
|
||||||
|
format: Option<String>,
|
||||||
|
files: Vec<SubtitleFileRef>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SubtitleFileRef {
|
||||||
|
file_id: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The API spells languages `pt-PT`, `pt-BR`, `en`; accept `_` too, since
|
||||||
|
/// tooling around this API uses it.
|
||||||
|
fn language_of(code: &str) -> Option<Language> {
|
||||||
|
match code.replace('_', "-").to_ascii_lowercase().as_str() {
|
||||||
|
"pt-pt" => Some(Language::PortuguesePortugal),
|
||||||
|
"pt-br" => Some(Language::PortugueseBrazil),
|
||||||
|
"pt" | "por" => Some(Language::PortugueseUnverified),
|
||||||
|
"" => None,
|
||||||
|
_ => Some(Language::Other(code.to_owned())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `languages` query parameter: comma-separated codes, pt-PT and pt-BR
|
||||||
|
/// kept apart — keeping them apart is why this is the primary provider.
|
||||||
|
fn languages_param(languages: &[Language]) -> String {
|
||||||
|
languages
|
||||||
|
.iter()
|
||||||
|
.map(|language| match language {
|
||||||
|
Language::PortuguesePortugal => "pt-PT",
|
||||||
|
Language::PortugueseBrazil => "pt-BR",
|
||||||
|
Language::PortugueseUnverified => "pt",
|
||||||
|
Language::Other(tag) => tag.as_str(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The release group of the release a subtitle was timed against, read off
|
||||||
|
/// the release name with the same parser the indexer uses.
|
||||||
|
fn group_of_release(release: &str) -> Option<String> {
|
||||||
|
arr_parse::parse(release).group
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_of_release(release: &str) -> Option<arr_core::Source> {
|
||||||
|
arr_parse::parse(release).source.map(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_of(format: Option<&str>) -> SubtitleFormat {
|
||||||
|
match format.unwrap_or_default().to_ascii_lowercase().as_str() {
|
||||||
|
"srt" | "subrip" => SubtitleFormat::Srt,
|
||||||
|
"ass" | "ssa" => SubtitleFormat::Ass,
|
||||||
|
"vtt" | "webvtt" => SubtitleFormat::Vtt,
|
||||||
|
other => SubtitleFormat::Other(other.to_owned()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Order the candidates best first through the pure ranker (#185), building
|
||||||
|
/// the target from the same facts the search ran on: the file's own hash and
|
||||||
|
/// whatever the release name claims about group and source.
|
||||||
|
fn rank_candidates(
|
||||||
|
candidates: Vec<Candidate>,
|
||||||
|
file: &MediaFile,
|
||||||
|
target_hash: Option<&str>,
|
||||||
|
) -> Vec<Candidate> {
|
||||||
|
let claims = file.release_name.as_deref().map(arr_parse::parse);
|
||||||
|
let target = arr_core::subs::SubtitleTarget {
|
||||||
|
moviehash: target_hash,
|
||||||
|
release_name: file.release_name.as_deref(),
|
||||||
|
release_group: claims.as_ref().and_then(|claims| claims.group.as_deref()),
|
||||||
|
source: claims
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|claims| claims.source.map(Into::into)),
|
||||||
|
};
|
||||||
|
|
||||||
|
let cores: Vec<_> = candidates.iter().map(|c| c.to_core(target_hash)).collect();
|
||||||
|
let order: Vec<usize> = arr_core::subs::rank(&target, &cores)
|
||||||
|
.iter()
|
||||||
|
.map(|entry| entry.index)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut slots: Vec<_> = candidates.into_iter().map(Some).collect();
|
||||||
|
order
|
||||||
|
.iter()
|
||||||
|
.map(|&index| slots[index].take().expect("each index ranks once"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{format_of, language_of, moviehash, rank_candidates, PROVIDER_NAME};
|
||||||
|
use crate::{Candidate, CandidateId, MediaFile, MediaRef, ProviderId, SubtitleFormat};
|
||||||
|
use arr_core::Language;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_zero_filled_file_hashes_to_its_own_size() {
|
||||||
|
// Every 8-byte word of both chunks sums to zero, so the whole hash is
|
||||||
|
// the file size: 262144 = 0x40000. Hand-derived from the algorithm,
|
||||||
|
// not computed with the code under test.
|
||||||
|
let dir = tempfile::tempdir().expect("test setup");
|
||||||
|
let path = dir.path().join("zeros.mkv");
|
||||||
|
std::fs::write(&path, vec![0u8; 262_144]).expect("test setup");
|
||||||
|
|
||||||
|
let hash = moviehash(&path, 262_144).expect("test setup");
|
||||||
|
|
||||||
|
assert_eq!(hash.as_deref(), Some("0000000000040000"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_last_chunk_is_read_from_the_end_not_the_front() {
|
||||||
|
// First chunk all 0xFF: 8192 words of -1 sum to -8192. Last chunk all
|
||||||
|
// zero. Size 131072 - 8192 = 122880 = 0x1e000. If both ends were read
|
||||||
|
// from the front the sum would double instead.
|
||||||
|
let dir = tempfile::tempdir().expect("test setup");
|
||||||
|
let path = dir.path().join("ends.mkv");
|
||||||
|
let first = vec![0xFFu8; 65_536];
|
||||||
|
let rest = vec![0u8; 65_536];
|
||||||
|
let mut bytes = first;
|
||||||
|
bytes.extend_from_slice(&rest);
|
||||||
|
std::fs::write(&path, &bytes).expect("test setup");
|
||||||
|
|
||||||
|
let hash = moviehash(&path, 131_072).expect("test setup");
|
||||||
|
|
||||||
|
assert_eq!(hash.as_deref(), Some("000000000001e000"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_file_too_short_for_two_chunks_has_no_hash() {
|
||||||
|
let dir = tempfile::tempdir().expect("test setup");
|
||||||
|
let path = dir.path().join("small.mkv");
|
||||||
|
std::fs::write(&path, vec![0u8; 65_536]).expect("test setup");
|
||||||
|
|
||||||
|
assert_eq!(moviehash(&path, 65_536).expect("test setup"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn languages_map_both_ways_with_pt_variants_apart() {
|
||||||
|
assert_eq!(language_of("pt-PT"), Some(Language::PortuguesePortugal));
|
||||||
|
assert_eq!(language_of("pt_BR"), Some(Language::PortugueseBrazil));
|
||||||
|
assert_eq!(language_of("por"), Some(Language::PortugueseUnverified));
|
||||||
|
assert_eq!(language_of("en"), Some(Language::Other("en".to_owned())));
|
||||||
|
assert_eq!(language_of(""), None);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
super::languages_param(&[
|
||||||
|
Language::PortuguesePortugal,
|
||||||
|
Language::PortugueseBrazil,
|
||||||
|
Language::PortugueseUnverified,
|
||||||
|
Language::Other("en".to_owned()),
|
||||||
|
]),
|
||||||
|
"pt-PT,pt-BR,pt,en"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn formats_spell_the_way_providers_do() {
|
||||||
|
assert_eq!(format_of(Some("srt")), SubtitleFormat::Srt);
|
||||||
|
assert_eq!(format_of(Some("subrip")), SubtitleFormat::Srt);
|
||||||
|
assert_eq!(format_of(Some("ass")), SubtitleFormat::Ass);
|
||||||
|
assert_eq!(format_of(Some("vtt")), SubtitleFormat::Vtt);
|
||||||
|
assert_eq!(
|
||||||
|
format_of(Some("idx")),
|
||||||
|
SubtitleFormat::Other("idx".to_owned())
|
||||||
|
);
|
||||||
|
assert_eq!(format_of(None), SubtitleFormat::Other(String::new()));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn candidate(id: u64, hash_match: bool, downloads: u64) -> Candidate {
|
||||||
|
Candidate {
|
||||||
|
provider: ProviderId::new(PROVIDER_NAME),
|
||||||
|
id: CandidateId::new(id.to_string()),
|
||||||
|
language: Language::PortuguesePortugal,
|
||||||
|
hash_match,
|
||||||
|
release_name: None,
|
||||||
|
group: None,
|
||||||
|
source: None,
|
||||||
|
rating: None,
|
||||||
|
download_count: Some(downloads),
|
||||||
|
forced: false,
|
||||||
|
sdh: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn search_results_come_back_ranked_best_first() {
|
||||||
|
let file = MediaFile {
|
||||||
|
path: "/m/f.mkv".into(),
|
||||||
|
size: 1 << 20,
|
||||||
|
release_name: None,
|
||||||
|
media: MediaRef::Movie { tmdb_id: 42 },
|
||||||
|
};
|
||||||
|
// The hash-matched candidate has fewer downloads; ranking puts it
|
||||||
|
// first anyway, because a moviehash match wins outright.
|
||||||
|
let candidates = vec![candidate(1, false, 9_999), candidate(2, true, 1)];
|
||||||
|
|
||||||
|
let ranked = rank_candidates(candidates, &file, Some("abc123"));
|
||||||
|
|
||||||
|
let ids: Vec<_> = ranked.iter().map(|c| c.id.as_str()).collect();
|
||||||
|
assert_eq!(ids, ["2", "1"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"total_count": "1",
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"id": "7111111",
|
||||||
|
"type": "subtitle",
|
||||||
|
"attributes": {
|
||||||
|
"subtitle_id": "7111111",
|
||||||
|
"language": "pt-PT",
|
||||||
|
"release": "Series.S01E02.1080p.WEB-DL-GRP",
|
||||||
|
"moviehash_match": false,
|
||||||
|
"ratings": 60.0,
|
||||||
|
"download_count": 77,
|
||||||
|
"hearing_impaired": false,
|
||||||
|
"foreign_parts_only": false,
|
||||||
|
"format": "ass",
|
||||||
|
"files": [
|
||||||
|
{ "file_id": 61111111, "file_name": "series.s01e02.pt.ass", "cd_number": 1 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
{
|
||||||
|
"total_count": "3",
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"id": "7013923",
|
||||||
|
"type": "subtitle",
|
||||||
|
"attributes": {
|
||||||
|
"subtitle_id": "7013923",
|
||||||
|
"language": "pt-PT",
|
||||||
|
"release": "Dune.Part.Two.2024.1080p.WEB-DL.DDP5.1.Atmos.H.264-GRP",
|
||||||
|
"moviehash_match": true,
|
||||||
|
"ratings": 82.0,
|
||||||
|
"download_count": 12345,
|
||||||
|
"hearing_impaired": false,
|
||||||
|
"foreign_parts_only": false,
|
||||||
|
"format": "srt",
|
||||||
|
"uploader": { "name": "someone", "rating": 90 },
|
||||||
|
"files": [
|
||||||
|
{ "file_id": 60619911, "file_name": "Dune.Part.Two.2024.1080p.WEB-DL-GRP.pt.srt", "cd_number": 1 }
|
||||||
|
],
|
||||||
|
"feature_details": { "year": 2024, "title": "Dune: Part Two", "tmdb_id": 693134 }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "7000001",
|
||||||
|
"type": "subtitle",
|
||||||
|
"attributes": {
|
||||||
|
"subtitle_id": "7000001",
|
||||||
|
"language": "pt-BR",
|
||||||
|
"release": "Dune.Part.Two.2024.2160p.BluRay.GRP",
|
||||||
|
"moviehash_match": false,
|
||||||
|
"ratings": 95.0,
|
||||||
|
"download_count": 999999,
|
||||||
|
"hearing_impaired": false,
|
||||||
|
"foreign_parts_only": false,
|
||||||
|
"format": "srt",
|
||||||
|
"uploader": { "name": "other", "rating": 100 },
|
||||||
|
"files": [
|
||||||
|
{ "file_id": 60000001, "file_name": "dune.part.two.br.srt", "cd_number": 1 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "6999999",
|
||||||
|
"type": "subtitle",
|
||||||
|
"attributes": {
|
||||||
|
"subtitle_id": "6999999",
|
||||||
|
"language": "en",
|
||||||
|
"release": "Dune.Part.Two.2024.1080p.WEB-DL.GRP",
|
||||||
|
"moviehash_match": true,
|
||||||
|
"ratings": 40.0,
|
||||||
|
"download_count": 5,
|
||||||
|
"hearing_impaired": true,
|
||||||
|
"foreign_parts_only": false,
|
||||||
|
"format": "srt",
|
||||||
|
"files": [
|
||||||
|
{ "file_id": 59999991, "file_name": "dune.en.sdh.srt", "cd_number": 1 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "6888888",
|
||||||
|
"type": "subtitle",
|
||||||
|
"attributes": {
|
||||||
|
"subtitle_id": "6888888",
|
||||||
|
"language": "fr",
|
||||||
|
"release": "Dune.Part.Two.2024.1080p.WEB-DL.GRP",
|
||||||
|
"moviehash_match": false,
|
||||||
|
"ratings": null,
|
||||||
|
"download_count": 12,
|
||||||
|
"hearing_impaired": false,
|
||||||
|
"foreign_parts_only": true,
|
||||||
|
"format": "srt",
|
||||||
|
"files": [
|
||||||
|
{ "file_id": 58888881, "file_name": "dune.fr.forced.srt", "cd_number": 1 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,423 @@
|
|||||||
|
//! OpenSubtitles.com provider tests. Everything runs against `wiremock` —
|
||||||
|
//! the issue rules out pointing tests at the live service, which would
|
||||||
|
//! burn the daily download cap and leak a key into CI.
|
||||||
|
|
||||||
|
// Same per-target quirk as in `lib.rs`: an integration test links the library's
|
||||||
|
// dependencies without using them directly.
|
||||||
|
use {arr_parse as _, reqwest as _, serde as _, thiserror as _, tracing as _};
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use arr_core::Language;
|
||||||
|
use arr_subs::{
|
||||||
|
CandidateId, Error, MediaFile, MediaRef, OpenSubtitles, OpenSubtitlesConfig, Provider,
|
||||||
|
SearchRequest, SubtitleFormat,
|
||||||
|
};
|
||||||
|
use tempfile::TempDir;
|
||||||
|
use wiremock::matchers::{body_partial_json, header, method, path, query_param};
|
||||||
|
use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate};
|
||||||
|
|
||||||
|
const SEARCH_MOVIE: &str = include_str!("fixtures/subtitles_search_movie.json");
|
||||||
|
const SEARCH_EPISODE: &str = include_str!("fixtures/subtitles_search_episode.json");
|
||||||
|
|
||||||
|
fn config() -> OpenSubtitlesConfig {
|
||||||
|
OpenSubtitlesConfig {
|
||||||
|
api_key: "test-api-key".to_owned(),
|
||||||
|
username: Some("user".to_owned()),
|
||||||
|
password: Some("pass".to_owned()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn client(server: &MockServer) -> OpenSubtitles {
|
||||||
|
OpenSubtitles::with_base_url(config(), &format!("{}/api/v1/", server.uri()))
|
||||||
|
.expect("client builds")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn anonymous_client(server: &MockServer) -> OpenSubtitles {
|
||||||
|
OpenSubtitles::with_base_url(
|
||||||
|
OpenSubtitlesConfig {
|
||||||
|
api_key: "test-api-key".to_owned(),
|
||||||
|
username: None,
|
||||||
|
password: None,
|
||||||
|
},
|
||||||
|
&format!("{}/api/v1/", server.uri()),
|
||||||
|
)
|
||||||
|
.expect("client builds")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 256 KiB of zeros hashes to its own size — see the unit tests in the
|
||||||
|
/// module — so every search against this file sends
|
||||||
|
/// `moviehash=0000000000040000`.
|
||||||
|
struct HashedFile(TempDir);
|
||||||
|
|
||||||
|
impl HashedFile {
|
||||||
|
fn new() -> Self {
|
||||||
|
let dir = TempDir::new().expect("tempdir");
|
||||||
|
let path = dir.path().join("dune.mkv");
|
||||||
|
std::fs::write(&path, vec![0u8; 262_144]).expect("write");
|
||||||
|
Self(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn media(&self) -> MediaFile {
|
||||||
|
MediaFile {
|
||||||
|
path: self.0.path().join("dune.mkv"),
|
||||||
|
size: 262_144,
|
||||||
|
release_name: Some("Dune.Part.Two.2024.1080p.WEB-DL.GRP".to_owned()),
|
||||||
|
media: MediaRef::Movie { tmdb_id: 693_134 },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request(file: MediaFile, languages: Vec<Language>) -> SearchRequest {
|
||||||
|
SearchRequest { file, languages }
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mount_search(server: &MockServer, body: &'static str) {
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/subtitles"))
|
||||||
|
.and(header("Api-Key", "test-api-key"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(body))
|
||||||
|
.mount(server)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn movie_search_sends_both_lanes_and_ranks_hash_match_first() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let file = HashedFile::new();
|
||||||
|
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/subtitles"))
|
||||||
|
.and(header("Api-Key", "test-api-key"))
|
||||||
|
.and(query_param("tmdb_movie_id", "693134"))
|
||||||
|
.and(query_param("moviehash", "0000000000040000"))
|
||||||
|
.and(query_param("languages", "pt-PT,pt-BR,en"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(SEARCH_MOVIE))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let candidates = client(&server)
|
||||||
|
.search(&request(
|
||||||
|
file.media(),
|
||||||
|
vec![
|
||||||
|
Language::PortuguesePortugal,
|
||||||
|
Language::PortugueseBrazil,
|
||||||
|
Language::Other("en".to_owned()),
|
||||||
|
],
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds");
|
||||||
|
|
||||||
|
// The French entry is not a wanted language and is gone; pt-PT, pt-BR
|
||||||
|
// and English remain. The hash-matched pt-PT candidate outranks the
|
||||||
|
// pt-BR one despite ten thousand times fewer downloads, because a
|
||||||
|
// moviehash match wins outright (§15).
|
||||||
|
assert_eq!(candidates.len(), 3);
|
||||||
|
assert_eq!(candidates[0].id.as_str(), "60619911");
|
||||||
|
assert_eq!(candidates[0].language, Language::PortuguesePortugal);
|
||||||
|
assert!(candidates[0].hash_match);
|
||||||
|
|
||||||
|
// Facts #185 ranks on survive the mapping.
|
||||||
|
assert_eq!(
|
||||||
|
candidates[0].release_name.as_deref(),
|
||||||
|
Some("Dune.Part.Two.2024.1080p.WEB-DL.DDP5.1.Atmos.H.264-GRP")
|
||||||
|
);
|
||||||
|
assert_eq!(candidates[0].group.as_deref(), Some("GRP"));
|
||||||
|
assert_eq!(candidates[1].download_count, Some(999_999));
|
||||||
|
// A plain subtitle outranks an SDH one for the same language — here even
|
||||||
|
// against another moviehash match, because plainness is the first tier.
|
||||||
|
assert!(candidates[2].hash_match);
|
||||||
|
assert!(candidates[2].sdh);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn episode_search_addresses_the_series_not_the_movie_lane() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/subtitles"))
|
||||||
|
.and(query_param("tmdb_series_id", "94605"))
|
||||||
|
.and(query_param("season_number", "3"))
|
||||||
|
.and(query_param("episode_number", "7"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(SEARCH_EPISODE))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let file = MediaFile {
|
||||||
|
// Below the two-chunk minimum, so the hash lane stays out and no
|
||||||
|
// real file needs to exist on disk.
|
||||||
|
path: "/m/series.mkv".into(),
|
||||||
|
size: 65_536,
|
||||||
|
release_name: None,
|
||||||
|
media: MediaRef::Episode {
|
||||||
|
tmdb_id: 94_605,
|
||||||
|
season: 3,
|
||||||
|
episode: 7,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let candidates = client(&server)
|
||||||
|
.search(&request(file, vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds");
|
||||||
|
|
||||||
|
assert_eq!(candidates.len(), 1);
|
||||||
|
assert_eq!(candidates[0].id.as_str(), "61111111");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Matches a request where the named query parameter is absent entirely.
|
||||||
|
struct QueryParamMissing(&'static str);
|
||||||
|
|
||||||
|
impl Match for QueryParamMissing {
|
||||||
|
fn matches(&self, request: &Request) -> bool {
|
||||||
|
!request.url.query_pairs().any(|(key, _)| key == self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_file_too_small_to_hash_searches_without_the_hash_lane() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/subtitles"))
|
||||||
|
.and(query_param("tmdb_movie_id", "42"))
|
||||||
|
.and(QueryParamMissing("moviehash"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(SEARCH_EPISODE))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let file = MediaFile {
|
||||||
|
path: "/m/small.mkv".into(),
|
||||||
|
size: 65_536,
|
||||||
|
release_name: None,
|
||||||
|
media: MediaRef::Movie { tmdb_id: 42 },
|
||||||
|
};
|
||||||
|
let candidates = client(&server)
|
||||||
|
.search(&request(file, vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds");
|
||||||
|
|
||||||
|
assert_eq!(candidates.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_expired_key_is_unauthorized_and_a_cap_is_rate_limited() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let file = HashedFile::new();
|
||||||
|
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/subtitles"))
|
||||||
|
.and(header("Api-Key", "wrong-key"))
|
||||||
|
.respond_with(ResponseTemplate::new(401))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/api/v1/subtitles"))
|
||||||
|
.and(header("Api-Key", "test-api-key"))
|
||||||
|
.respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "3600"))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let wrong_key = OpenSubtitles::with_base_url(
|
||||||
|
OpenSubtitlesConfig {
|
||||||
|
api_key: "wrong-key".to_owned(),
|
||||||
|
username: None,
|
||||||
|
password: None,
|
||||||
|
},
|
||||||
|
&format!("{}/api/v1/", server.uri()),
|
||||||
|
)
|
||||||
|
.expect("client builds");
|
||||||
|
let err = wrong_key
|
||||||
|
.search(&request(file.media(), vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect_err("bad key is refused");
|
||||||
|
assert!(matches!(err, Error::Unauthorized { .. }));
|
||||||
|
|
||||||
|
let err = client(&server)
|
||||||
|
.search(&request(file.media(), vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect_err("cap hit");
|
||||||
|
match err {
|
||||||
|
Error::RateLimited { retry_after, .. } => {
|
||||||
|
assert_eq!(retry_after, Some(Duration::from_hours(1)));
|
||||||
|
}
|
||||||
|
other => panic!("expected RateLimited, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mount_login(server: &MockServer, token: &'static str) {
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/api/v1/login"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200).set_body_json(serde_json::json!({ "token": token })),
|
||||||
|
)
|
||||||
|
.mount(server)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mount_download_link(server: &MockServer, token: &'static str, file_id: u64) {
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/api/v1/download"))
|
||||||
|
.and(header("Authorization", format!("Bearer {token}")))
|
||||||
|
.and(body_partial_json(serde_json::json!({ "file_id": file_id })))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"link": format!("{}/file/{file_id}.srt", server.uri()),
|
||||||
|
"file_name": "subtitle.srt",
|
||||||
|
"requests": 99
|
||||||
|
})))
|
||||||
|
.mount(server)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn download_logs_in_then_walks_link_and_bytes_back_into_fetched() {
|
||||||
|
const SRT: &[u8] = b"1\n00:00:01,000 --> 00:00:02,000\nol\xc3\xa1\n";
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
|
||||||
|
let file = HashedFile::new();
|
||||||
|
mount_search(&server, SEARCH_MOVIE).await;
|
||||||
|
mount_login(&server, "token-1").await;
|
||||||
|
mount_download_link(&server, "token-1", 60_619_911).await;
|
||||||
|
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/file/60619911.srt"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_bytes(SRT.to_vec()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let provider = client(&server);
|
||||||
|
provider
|
||||||
|
.search(&request(file.media(), vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds");
|
||||||
|
|
||||||
|
let fetched = provider
|
||||||
|
.download(&CandidateId::new("60619911"))
|
||||||
|
.await
|
||||||
|
.expect("download succeeds");
|
||||||
|
|
||||||
|
assert_eq!(fetched.id.as_str(), "60619911");
|
||||||
|
assert_eq!(fetched.language, Language::PortuguesePortugal);
|
||||||
|
assert_eq!(fetched.format, SubtitleFormat::Srt);
|
||||||
|
assert_eq!(fetched.content, SRT);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_expired_token_refreshes_once_and_retries() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let file = HashedFile::new();
|
||||||
|
mount_search(&server, SEARCH_MOVIE).await;
|
||||||
|
|
||||||
|
// The first login hands out the stale token; the refresh after the 401
|
||||||
|
// gets the fresh one. `up_to_times(1)` retires the first mock so the
|
||||||
|
// second call falls through to the next.
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/api/v1/login"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200).set_body_json(serde_json::json!({ "token": "stale" })),
|
||||||
|
)
|
||||||
|
.up_to_n_times(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/api/v1/login"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200).set_body_json(serde_json::json!({ "token": "fresh" })),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/api/v1/download"))
|
||||||
|
.and(header("Authorization", "Bearer stale"))
|
||||||
|
.respond_with(ResponseTemplate::new(401))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
mount_download_link(&server, "fresh", 60_619_911).await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/file/60619911.srt"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"sub".to_vec()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let provider = client(&server);
|
||||||
|
provider
|
||||||
|
.search(&request(file.media(), vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds");
|
||||||
|
|
||||||
|
let fetched = provider
|
||||||
|
.download(&CandidateId::new("60619911"))
|
||||||
|
.await
|
||||||
|
.expect("the refreshed token downloads");
|
||||||
|
|
||||||
|
assert_eq!(fetched.content, b"sub");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn hitting_the_download_cap_is_rate_limited_not_broken() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let file = HashedFile::new();
|
||||||
|
mount_search(&server, SEARCH_MOVIE).await;
|
||||||
|
mount_login(&server, "t").await;
|
||||||
|
|
||||||
|
// 406 is what the API documents for the daily-download cap.
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/api/v1/download"))
|
||||||
|
.respond_with(ResponseTemplate::new(406).insert_header("Retry-After", "86400"))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let provider = client(&server);
|
||||||
|
provider
|
||||||
|
.search(&request(file.media(), vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds");
|
||||||
|
|
||||||
|
let err = provider
|
||||||
|
.download(&CandidateId::new("60619911"))
|
||||||
|
.await
|
||||||
|
.expect_err("cap reached");
|
||||||
|
match err {
|
||||||
|
Error::RateLimited { retry_after, .. } => {
|
||||||
|
assert_eq!(retry_after, Some(Duration::from_hours(24)));
|
||||||
|
}
|
||||||
|
other => panic!("expected RateLimited, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn downloading_without_user_credentials_never_reaches_the_wire() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let file = HashedFile::new();
|
||||||
|
mount_search(&server, SEARCH_MOVIE).await;
|
||||||
|
|
||||||
|
let provider = anonymous_client(&server);
|
||||||
|
provider
|
||||||
|
.search(&request(file.media(), vec![Language::PortuguesePortugal]))
|
||||||
|
.await
|
||||||
|
.expect("search succeeds anonymously");
|
||||||
|
|
||||||
|
let err = provider
|
||||||
|
.download(&CandidateId::new("60619911"))
|
||||||
|
.await
|
||||||
|
.expect_err("no user configured");
|
||||||
|
|
||||||
|
assert!(matches!(err, Error::Unauthorized { .. }));
|
||||||
|
// No /login or /download request was ever mounted, so reaching for them
|
||||||
|
// would have failed the test with an unhandled-request error instead.
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_id_this_provider_never_offered_is_not_found() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
|
||||||
|
let err = client(&server)
|
||||||
|
.download(&CandidateId::new("99999999"))
|
||||||
|
.await
|
||||||
|
.expect_err("unknown id");
|
||||||
|
|
||||||
|
match err {
|
||||||
|
Error::NotFound { candidate, .. } => assert_eq!(candidate.as_str(), "99999999"),
|
||||||
|
other => panic!("expected NotFound, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user