feat(arr): fetch subtitles from Podnapisi
This commit is contained in:
Miguel Palhas
@@ -20,8 +20,15 @@
|
||||
|
||||
use std::{fmt, future::Future, pin::Pin};
|
||||
|
||||
// `unused_crate_dependencies` is a per-target lint; `wiremock` is only used
|
||||
// from the `tests/podnapisi.rs` integration target, not from this library
|
||||
// target's own `#[cfg(test)]` module.
|
||||
#[cfg(test)]
|
||||
use wiremock as _;
|
||||
|
||||
pub mod error;
|
||||
pub mod model;
|
||||
pub mod podnapisi;
|
||||
pub mod srt;
|
||||
pub mod translate;
|
||||
|
||||
@@ -29,6 +36,9 @@ pub use error::{Error, Result};
|
||||
pub use model::{
|
||||
Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat,
|
||||
};
|
||||
pub use podnapisi::{
|
||||
PodnapisiProvider, PodnapisiProviderBuilder, DEFAULT_BASE_URL as PODNAPISI_DEFAULT_BASE_URL,
|
||||
};
|
||||
pub use srt::Cue;
|
||||
pub use translate::{Backend, BackendId, Batch, BatchCue, TranslateFuture, TranslatedCue};
|
||||
|
||||
|
||||
@@ -0,0 +1,490 @@
|
||||
//! The Podnapisi.net provider (`DESIGN.md` §15).
|
||||
//!
|
||||
//! Podnapisi has no `moviehash` lane: it matches on a text search over the
|
||||
//! release name it was imported under, nothing else. Every candidate this
|
||||
//! provider returns therefore carries `hash_match: false`, which is exactly
|
||||
//! what ranking (#185) already expresses without a special case — a plain
|
||||
//! release-name match and below.
|
||||
//!
|
||||
//! The search API is unauthenticated JSON, and download hands back a zip
|
||||
//! containing exactly one subtitle file. Both are undocumented officially;
|
||||
//! this follows the shape `subliminal`'s maintained provider uses against the
|
||||
//! live service.
|
||||
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
io::{Cursor, Read as _},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use arr_core::{
|
||||
lang::{resolve_portuguese, PortugueseEvidence},
|
||||
Language,
|
||||
};
|
||||
use reqwest::{StatusCode, Url};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{
|
||||
error::{Error, Result},
|
||||
model::{
|
||||
Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest,
|
||||
SubtitleFormat,
|
||||
},
|
||||
DownloadFuture, Provider, SearchFuture,
|
||||
};
|
||||
|
||||
/// Podnapisi's public search and download root.
|
||||
pub const DEFAULT_BASE_URL: &str = "https://www.podnapisi.net/subtitles/";
|
||||
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// A results page claiming more pages than this is treated as malformed
|
||||
/// rather than looped over forever.
|
||||
const MAX_PAGES: u32 = 50;
|
||||
|
||||
fn provider_id() -> ProviderId {
|
||||
ProviderId::new("podnapisi")
|
||||
}
|
||||
|
||||
/// The Podnapisi.net subtitle provider.
|
||||
#[derive(Debug)]
|
||||
pub struct PodnapisiProvider {
|
||||
http: reqwest::Client,
|
||||
base_url: Url,
|
||||
}
|
||||
|
||||
impl PodnapisiProvider {
|
||||
/// A provider against the real Podnapisi.net, with default timeouts.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`Error::Config`] if the HTTP client cannot be built.
|
||||
pub fn new() -> Result<Self> {
|
||||
Self::builder().build()
|
||||
}
|
||||
|
||||
/// Start configuring a provider.
|
||||
#[must_use]
|
||||
pub fn builder() -> PodnapisiProviderBuilder {
|
||||
PodnapisiProviderBuilder {
|
||||
base_url: DEFAULT_BASE_URL.to_owned(),
|
||||
timeout: DEFAULT_TIMEOUT,
|
||||
}
|
||||
}
|
||||
|
||||
async fn search_language(
|
||||
&self,
|
||||
file: &MediaFile,
|
||||
language: &Language,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
let Some(keywords) = file.release_name.as_deref() else {
|
||||
// No release name, nothing to search Podnapisi's keyword index
|
||||
// with. Not an error: the provider simply has nothing to offer.
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
let mut candidates = Vec::new();
|
||||
let mut seen_pids = HashSet::new();
|
||||
let mut page = 1;
|
||||
|
||||
loop {
|
||||
let response = self
|
||||
.fetch_page(keywords, language, &file.media, page)
|
||||
.await?;
|
||||
let all_pages = response.all_pages.max(1);
|
||||
|
||||
for item in response.data {
|
||||
if !seen_pids.insert(item.id.clone()) {
|
||||
continue;
|
||||
}
|
||||
candidates.push(item.into_candidate(language));
|
||||
}
|
||||
|
||||
if response.page >= all_pages || page >= MAX_PAGES {
|
||||
break;
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
async fn fetch_page(
|
||||
&self,
|
||||
keywords: &str,
|
||||
language: &Language,
|
||||
media: &MediaRef,
|
||||
page: u32,
|
||||
) -> Result<SearchResponse> {
|
||||
let mut url = self
|
||||
.base_url
|
||||
.join("search/advanced")
|
||||
.map_err(|err| config_error(format!("invalid search URL: {err}")))?;
|
||||
|
||||
{
|
||||
let mut params = url.query_pairs_mut();
|
||||
params.append_pair("keywords", keywords);
|
||||
params.append_pair("language", &language_param(language));
|
||||
match media {
|
||||
MediaRef::Movie { .. } => {
|
||||
params.append_pair("movie_type", "movie");
|
||||
}
|
||||
MediaRef::Episode {
|
||||
season, episode, ..
|
||||
} => {
|
||||
params.append_pair("movie_type", "tv-series");
|
||||
params.append_pair("movie_type", "mini-series");
|
||||
params.append_pair("seasons", &season.to_string());
|
||||
params.append_pair("episodes", &episode.to_string());
|
||||
}
|
||||
}
|
||||
if page > 1 {
|
||||
params.append_pair("page", &page.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let body = self.get(url).await?;
|
||||
serde_json::from_str(&body).map_err(|err| Error::Malformed {
|
||||
provider: provider_id(),
|
||||
detail: format!("search response did not parse: {err}"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Download one candidate this provider offered.
|
||||
async fn download_candidate(&self, id: &CandidateId) -> Result<Fetched> {
|
||||
let (pid, language) = split_candidate_id(id)?;
|
||||
|
||||
let mut url = self
|
||||
.base_url
|
||||
.join(&format!("{pid}/download"))
|
||||
.map_err(|err| config_error(format!("invalid download URL: {err}")))?;
|
||||
url.query_pairs_mut().append_pair("container", "zip");
|
||||
|
||||
let bytes = self.get_bytes(url, id).await?;
|
||||
let (content, format) = extract_single_file(&bytes)?;
|
||||
|
||||
Ok(Fetched {
|
||||
id: id.clone(),
|
||||
language,
|
||||
format,
|
||||
content,
|
||||
})
|
||||
}
|
||||
|
||||
async fn get(&self, url: Url) -> Result<String> {
|
||||
let response = self.send(url, None).await?;
|
||||
response.text().await.map_err(|err| Error::Transport {
|
||||
provider: provider_id(),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_bytes(&self, url: Url, candidate: &CandidateId) -> Result<Vec<u8>> {
|
||||
let response = self.send(url, Some(candidate)).await?;
|
||||
response
|
||||
.bytes()
|
||||
.await
|
||||
.map(|bytes| bytes.to_vec())
|
||||
.map_err(|err| Error::Transport {
|
||||
provider: provider_id(),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
/// `candidate` is only known when the request is a download — it names
|
||||
/// the [`Error::NotFound`] a 404 becomes.
|
||||
async fn send(&self, url: Url, candidate: Option<&CandidateId>) -> Result<reqwest::Response> {
|
||||
tracing::debug!(url = %url, "Podnapisi request");
|
||||
let response = self
|
||||
.http
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| Error::Transport {
|
||||
provider: provider_id(),
|
||||
source: Box::new(err),
|
||||
})?;
|
||||
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
Err(match (status, candidate) {
|
||||
(StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN, _) => Error::Unauthorized {
|
||||
provider: provider_id(),
|
||||
},
|
||||
(StatusCode::TOO_MANY_REQUESTS, _) => Error::RateLimited {
|
||||
provider: provider_id(),
|
||||
retry_after: retry_after(&response),
|
||||
},
|
||||
(StatusCode::NOT_FOUND, Some(candidate)) => Error::NotFound {
|
||||
provider: provider_id(),
|
||||
candidate: candidate.clone(),
|
||||
},
|
||||
(other, _) => Error::Malformed {
|
||||
provider: provider_id(),
|
||||
detail: format!("HTTP {other}"),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn config_error(detail: String) -> Error {
|
||||
Error::Config {
|
||||
provider: provider_id(),
|
||||
detail,
|
||||
}
|
||||
}
|
||||
|
||||
impl Provider for PodnapisiProvider {
|
||||
fn id(&self) -> ProviderId {
|
||||
provider_id()
|
||||
}
|
||||
|
||||
fn search<'a>(&'a self, request: &'a SearchRequest) -> SearchFuture<'a> {
|
||||
Box::pin(async move {
|
||||
let mut candidates = Vec::new();
|
||||
for language in &request.languages {
|
||||
candidates.extend(self.search_language(&request.file, language).await?);
|
||||
}
|
||||
Ok(candidates)
|
||||
})
|
||||
}
|
||||
|
||||
fn download<'a>(&'a self, id: &'a CandidateId) -> DownloadFuture<'a> {
|
||||
Box::pin(async move { self.download_candidate(id).await })
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for a [`PodnapisiProvider`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PodnapisiProviderBuilder {
|
||||
base_url: String,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PodnapisiProviderBuilder {
|
||||
/// Point the provider somewhere other than Podnapisi.net. Tests use
|
||||
/// this; nothing else should.
|
||||
#[must_use]
|
||||
pub fn base_url(mut self, base_url: impl Into<String>) -> Self {
|
||||
self.base_url = base_url.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Per-request timeout.
|
||||
#[must_use]
|
||||
pub fn timeout(mut self, timeout: Duration) -> Self {
|
||||
self.timeout = timeout;
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the provider.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`Error::Config`] if the base URL will not parse or the HTTP client
|
||||
/// cannot be built.
|
||||
pub fn build(self) -> Result<PodnapisiProvider> {
|
||||
let mut base_url = self.base_url;
|
||||
if !base_url.ends_with('/') {
|
||||
base_url.push('/');
|
||||
}
|
||||
let base_url = Url::parse(&base_url).map_err(|err| Error::Config {
|
||||
provider: provider_id(),
|
||||
detail: format!("invalid base URL: {err}"),
|
||||
})?;
|
||||
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(self.timeout)
|
||||
.user_agent(concat!("arr/", env!("CARGO_PKG_VERSION")))
|
||||
.build()
|
||||
.map_err(|err| Error::Config {
|
||||
provider: provider_id(),
|
||||
detail: err.to_string(),
|
||||
})?;
|
||||
|
||||
Ok(PodnapisiProvider { http, base_url })
|
||||
}
|
||||
}
|
||||
|
||||
/// One page of Podnapisi's `search/advanced` response.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SearchResponse {
|
||||
data: Vec<RawCandidate>,
|
||||
page: u32,
|
||||
all_pages: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RawCandidate {
|
||||
id: String,
|
||||
language: String,
|
||||
#[serde(default)]
|
||||
flags: Vec<String>,
|
||||
#[serde(default)]
|
||||
releases: Vec<String>,
|
||||
#[serde(default)]
|
||||
custom_releases: Vec<String>,
|
||||
}
|
||||
|
||||
impl RawCandidate {
|
||||
fn into_candidate(self, requested: &Language) -> Candidate {
|
||||
let release_name = self.releases.into_iter().chain(self.custom_releases).next();
|
||||
let claims = release_name.as_deref().map(arr_parse::parse);
|
||||
|
||||
let language = resolve_language(
|
||||
&self.language,
|
||||
claims.as_ref().map_or(&[], |claims| &claims.languages[..]),
|
||||
requested,
|
||||
);
|
||||
|
||||
Candidate {
|
||||
provider: provider_id(),
|
||||
id: CandidateId::new(format!("{}|{language}", self.id)),
|
||||
language,
|
||||
hash_match: false,
|
||||
group: claims.as_ref().and_then(|claims| claims.group.clone()),
|
||||
source: claims.and_then(|claims| claims.source).map(Into::into),
|
||||
release_name,
|
||||
rating: None,
|
||||
download_count: None,
|
||||
forced: false,
|
||||
sdh: self.flags.iter().any(|flag| flag == "hearing_impaired"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Turn a wanted [`Language`] into the string Podnapisi's `language` query
|
||||
/// parameter expects — the same BCP-47-shaped spelling [`Language::Display`]
|
||||
/// already produces for the pt-PT/pt-BR split, lower-cased where this crate
|
||||
/// has no opinion.
|
||||
fn language_param(language: &Language) -> String {
|
||||
match language {
|
||||
Language::PortuguesePortugal => "pt-PT".to_owned(),
|
||||
Language::PortugueseBrazil => "pt-BR".to_owned(),
|
||||
Language::PortugueseUnverified => "pt".to_owned(),
|
||||
Language::Other(tag) => tag.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve Podnapisi's own `language` tag on a result, using the release
|
||||
/// name's claimed language markers as the first signal — the same signal
|
||||
/// order `arr-core::lang` documents, with the container tag Podnapisi sends
|
||||
/// standing in for a BCP-47 tag.
|
||||
fn resolve_language(
|
||||
raw_tag: &str,
|
||||
name_markers: &[arr_parse::LanguageMarker],
|
||||
requested: &Language,
|
||||
) -> Language {
|
||||
let primary = raw_tag
|
||||
.split(['-', '_'])
|
||||
.next()
|
||||
.unwrap_or(raw_tag)
|
||||
.to_ascii_lowercase();
|
||||
|
||||
if primary == "pt" || primary == "por" {
|
||||
return resolve_portuguese(PortugueseEvidence {
|
||||
name_markers,
|
||||
stream_title: None,
|
||||
handler_name: None,
|
||||
container_tag: Some(raw_tag),
|
||||
});
|
||||
}
|
||||
|
||||
// Podnapisi echoes back whatever it matched the search on; when the tag
|
||||
// carries no useful primary subtag, the language we asked for is a
|
||||
// better answer than a guess.
|
||||
if primary.is_empty() {
|
||||
return requested.clone();
|
||||
}
|
||||
|
||||
Language::Other(primary)
|
||||
}
|
||||
|
||||
/// The `CandidateId` this provider issues is `{pid}|{language}`, the
|
||||
/// language spelled exactly as [`Language::Display`] renders it — so
|
||||
/// `download` recovers it without a second round trip to disambiguate
|
||||
/// pt-PT from pt-BR.
|
||||
fn split_candidate_id(id: &CandidateId) -> Result<(&str, Language)> {
|
||||
id.as_str()
|
||||
.rsplit_once('|')
|
||||
.map(|(pid, language)| (pid, parse_language_tag(language)))
|
||||
.ok_or_else(|| Error::NotFound {
|
||||
provider: provider_id(),
|
||||
candidate: id.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The inverse of [`Language::Display`], for the tag this module itself
|
||||
/// wrote into a [`CandidateId`]. Not a general parser: it only has to
|
||||
/// round-trip the four spellings `Display` can produce.
|
||||
fn parse_language_tag(tag: &str) -> Language {
|
||||
match tag {
|
||||
"pt-PT" => Language::PortuguesePortugal,
|
||||
"pt-BR" => Language::PortugueseBrazil,
|
||||
"por-unverified" => Language::PortugueseUnverified,
|
||||
other => Language::Other(other.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_after(response: &reqwest::Response) -> Option<Duration> {
|
||||
response
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.trim()
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.map(Duration::from_secs)
|
||||
}
|
||||
|
||||
/// Unzip a Podnapisi download, which is always a single subtitle file.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`Error::Malformed`] if the body is not a zip, or does not contain
|
||||
/// exactly one file.
|
||||
fn extract_single_file(bytes: &[u8]) -> Result<(Vec<u8>, SubtitleFormat)> {
|
||||
let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|err| Error::Malformed {
|
||||
provider: provider_id(),
|
||||
detail: format!("download was not a zip: {err}"),
|
||||
})?;
|
||||
|
||||
if archive.len() != 1 {
|
||||
return Err(Error::Malformed {
|
||||
provider: provider_id(),
|
||||
detail: format!("zip contained {} files, expected 1", archive.len()),
|
||||
});
|
||||
}
|
||||
|
||||
let mut entry = archive.by_index(0).map_err(|err| Error::Malformed {
|
||||
provider: provider_id(),
|
||||
detail: format!("could not read the zipped subtitle: {err}"),
|
||||
})?;
|
||||
|
||||
let format =
|
||||
entry
|
||||
.name()
|
||||
.rsplit_once('.')
|
||||
.map_or(SubtitleFormat::Other(String::new()), |(_, ext)| {
|
||||
match ext.to_ascii_lowercase().as_str() {
|
||||
"srt" => SubtitleFormat::Srt,
|
||||
"ass" | "ssa" => SubtitleFormat::Ass,
|
||||
"vtt" => SubtitleFormat::Vtt,
|
||||
other => SubtitleFormat::Other(other.to_owned()),
|
||||
}
|
||||
});
|
||||
|
||||
let mut content = Vec::new();
|
||||
entry
|
||||
.read_to_end(&mut content)
|
||||
.map_err(|err| Error::Malformed {
|
||||
provider: provider_id(),
|
||||
detail: format!("could not read the zipped subtitle: {err}"),
|
||||
})?;
|
||||
|
||||
Ok((content, format))
|
||||
}
|
||||
Reference in New Issue
Block a user