From ef0fac4933287ca2580000a4addfde694c606701 Mon Sep 17 00:00:00 2001 From: Miguel Palhas Date: Tue, 25 Aug 2026 01:00:41 +0100 Subject: [PATCH 1/3] feat(arr): let backends report billed characters --- crates/arr-subs/src/translate.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/arr-subs/src/translate.rs b/crates/arr-subs/src/translate.rs index 2fa8345..4114a90 100644 --- a/crates/arr-subs/src/translate.rs +++ b/crates/arr-subs/src/translate.rs @@ -128,6 +128,15 @@ pub trait Backend: fmt::Debug + Send + Sync { BATCH_CHAR_BUDGET } + /// Characters this backend has billed so far, cumulative over its life. + /// + /// Translators charge per character (DESIGN.md §15); the daily budget + /// (#197) spends against what was actually billed rather than an + /// estimate of it. A backend that cannot say reports zero. + fn characters_billed(&self) -> u64 { + 0 + } + /// Translate one batch. /// /// The reply is expected to carry every cue number the batch carried, From 0c07fbaed5b5b340244432f3f241a4b97208f822 Mon Sep 17 00:00:00 2001 From: Miguel Palhas Date: Tue, 25 Aug 2026 01:01:01 +0100 Subject: [PATCH 2/3] feat(arr): translate through the DeepL API --- crates/arr-subs/src/deepl.rs | 567 +++++++++++++++++++++++++++++++++++ crates/arr-subs/src/lib.rs | 22 +- 2 files changed, 583 insertions(+), 6 deletions(-) create mode 100644 crates/arr-subs/src/deepl.rs diff --git a/crates/arr-subs/src/deepl.rs b/crates/arr-subs/src/deepl.rs new file mode 100644 index 0000000..4920698 --- /dev/null +++ b/crates/arr-subs/src/deepl.rs @@ -0,0 +1,567 @@ +//! The `DeepL` translation backend (`DESIGN.md` §15, issue #192). +//! +//! `DeepL` is one of the two dedicated translation APIs §15 names. It is the +//! only backend that keeps pt-PT and pt-BR apart as first-class targets +//! (`PT-PT`, `PT-BR`), which is exactly why arr wants it — but it has no +//! plain-Portuguese target, so a request for generic `pt` is refused in +//! [`Backend::supports`] rather than silently answered with one variant or +//! the other. +//! +//! One request translates a whole batch: the texts go out as a JSON array +//! and come back as a same-length array of translations, order preserved, +//! so cue numbering never leaves [`crate::translate`]. Characters sent are +//! what `DeepL` bills, so they are counted and reported for #197's budget. +//! +//! The auth key arrives from bootstrap config or environment per §10 — +//! `ARR_TRANSLATE_DEEPL_API_KEY`, plumbed through `arr-daemon`'s config — +//! never a database row. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use reqwest::{Client, StatusCode, Url}; +use serde::{Deserialize, Serialize}; + +use arr_core::Language; + +use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue}; +use crate::{Error as SubsError, ProviderId, Result}; + +/// The `DeepL` API endpoint for keys ending in `:fx` (the free tier). +const FREE_BASE_URL: &str = "https://api-free.deepl.com/v2"; + +/// The `DeepL` API endpoint for every other key (the pro tier). +const PRO_BASE_URL: &str = "https://api.deepl.com/v2"; + +/// `DeepL`'s non-standard status for a spent billing-period quota. +const STATUS_QUOTA_EXCEEDED: u16 = 456; + +/// How much of an unexpected response body is worth keeping in an error. +const MAX_ERROR_BODY: usize = 512; + +/// Credentials for `DeepL` (DESIGN.md §10). +/// +/// A single auth key; free-tier keys end in `:fx` and select the +/// `api-free.deepl.com` host automatically. Never persisted to the database. +#[derive(Clone, PartialEq, Eq)] +pub struct DeepLConfig { + /// The `DeepL` auth key, sent as `DeepL-Auth-Key` on every call. + pub auth_key: String, +} + +// Hand-written: a derived Debug would print the key. +impl std::fmt::Debug for DeepLConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeepLConfig") + .field("auth_key", &"") + .finish() + } +} + +/// The `DeepL` client behind the [`Backend`] trait. +pub struct DeepL { + http: Client, + base_url: Url, + config: DeepLConfig, + id: BackendId, + billed: AtomicU64, +} + +// Hand-written: the key must never reach a log line. +impl std::fmt::Debug for DeepL { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeepL") + .field("base_url", &self.base_url.as_str()) + .finish_non_exhaustive() + } +} + +impl DeepL { + /// A client against the real `DeepL` API. The tier follows the key: one + /// ending in `:fx` talks to the free host, anything else to the pro host. + /// + /// # Errors + /// + /// When the HTTP client cannot be built or the derived base URL does not + /// parse. + pub fn new(config: DeepLConfig) -> Result { + let base = if config.auth_key.ends_with(":fx") { + FREE_BASE_URL + } else { + PRO_BASE_URL + }; + Self::with_base_url(config, base) + } + + /// A client against an arbitrary DeepL-compatible endpoint. Tests point + /// this at a local mock; an operator with a proxy can do the same. + /// + /// # Errors + /// + /// When the HTTP client cannot be built or `base_url` does not parse. + pub fn with_base_url(config: DeepLConfig, base_url: impl AsRef) -> Result { + let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config { + provider: ProviderId::new("deepl"), + detail: error.to_string(), + })?; + Ok(Self { + http: Client::new(), + base_url: url, + config, + id: BackendId::new("deepl"), + billed: AtomicU64::new(0), + }) + } + + fn join(&self, segment: &str) -> Url { + let mut url = self.base_url.clone(); + let path = format!("{}/{}", url.path().trim_end_matches('/'), segment); + url.set_path(&path); + url + } +} + +/// The language code `DeepL` expects, or `None` when it cannot express the +/// language exactly. +/// +/// Bare Portuguese is the case that matters: `DeepL` only accepts `PT-PT` and +/// `PT-BR` as a target, so `pt` on its own refuses rather than guessing a +/// variant. As a *source* hint bare `PT` is fine — a source describes text +/// arr already holds, it promises nothing about output. +#[must_use] +fn deep_code(language: &Language, allow_bare_pt: bool) -> Option { + let tag = match language { + Language::PortuguesePortugal => return Some("PT-PT".to_owned()), + Language::PortugueseBrazil => return Some("PT-BR".to_owned()), + Language::PortugueseUnverified => return None, + Language::Other(tag) => tag, + }; + + let mut parts = tag.split('-'); + let primary = parts.next()?; + if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic()) + { + return None; + } + let rest: Vec = parts.map(str::to_ascii_uppercase).collect(); + if rest.is_empty() && primary.eq_ignore_ascii_case("pt") && !allow_bare_pt { + return None; + } + + let mut out = primary.to_ascii_uppercase(); + for part in rest { + out.push('-'); + out.push_str(&part); + } + Some(out) +} + +#[derive(Serialize)] +struct Request<'a> { + text: &'a [String], + target_lang: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + source_lang: Option<&'a str>, +} + +#[derive(Deserialize)] +struct Response { + translations: Vec, +} + +#[derive(Deserialize)] +struct Translation { + #[serde(default)] + #[allow(dead_code)] + detected_source_language: Option, + text: String, +} + +impl Backend for DeepL { + fn id(&self) -> BackendId { + self.id.clone() + } + + fn supports(&self, target: &Language) -> bool { + deep_code(target, false).is_some() + } + + fn characters_billed(&self) -> u64 { + self.billed.load(Ordering::Relaxed) + } + + fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> { + Box::pin(async move { + let texts: Vec = batch.cues.iter().map(|cue| cue.text.clone()).collect(); + + // Unreachable when called through `translate`, which checks + // `supports` first; kept honest so a direct caller still refuses. + let Some(target_lang) = deep_code(&batch.target, false) else { + return Err(Error::UnsupportedTarget { + backend: self.id(), + language: batch.target.clone(), + }); + }; + let source_lang = deep_code(&batch.source, true); + + let reply = self + .http + .post(self.join("translate")) + .header("DeepL-Auth-Key", &self.config.auth_key) + .json(&Request { + text: &texts, + target_lang: &target_lang, + source_lang: source_lang.as_deref(), + }) + .send() + .await + .map_err(|error| Error::Transport { + backend: self.id(), + source: Box::new(error), + })?; + + let status = reply.status(); + match status { + StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }), + StatusCode::TOO_MANY_REQUESTS => { + let retry_after = retry_after(reply.headers()); + return Err(Error::RateLimited { + backend: self.id(), + retry_after, + }); + } + status if status.as_u16() == STATUS_QUOTA_EXCEEDED => { + // The billing-period character quota is spent. Same queue + // state as a rate limit (#197), just without a + // Retry-After to honour. + return Err(Error::RateLimited { + backend: self.id(), + retry_after: None, + }); + } + _ => {} + } + + let bytes = reply.bytes().await.map_err(|error| Error::Transport { + backend: self.id(), + source: Box::new(error), + })?; + if !status.is_success() { + let detail = String::from_utf8_lossy(&bytes); + if status.is_server_error() { + return Err(Error::Transport { + backend: self.id(), + source: format!("HTTP {status}: {}", truncate(&detail)).into(), + }); + } + return Err(Error::Malformed { + backend: self.id(), + detail: format!("HTTP {status}: {}", truncate(&detail)), + }); + } + + let parsed: Response = + serde_json::from_slice(&bytes).map_err(|error| Error::Malformed { + backend: self.id(), + detail: error.to_string(), + })?; + + if parsed.translations.len() != texts.len() { + return Err(Error::CueMismatch { + backend: self.id(), + detail: format!( + "sent {} texts, got {} translations back", + texts.len(), + parsed.translations.len() + ), + }); + } + + self.billed.fetch_add( + texts.iter().map(|text| text.chars().count() as u64).sum(), + Ordering::Relaxed, + ); + + Ok(parsed + .translations + .into_iter() + .zip(&batch.cues) + .map(|(translation, cue)| TranslatedCue { + number: cue.number, + text: translation.text, + }) + .collect()) + }) + } +} + +fn retry_after(headers: &reqwest::header::HeaderMap) -> Option { + headers + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(Duration::from_secs) +} + +fn truncate(body: &str) -> std::borrow::Cow<'_, str> { + if body.len() <= MAX_ERROR_BODY { + return std::borrow::Cow::Borrowed(body); + } + let mut end = MAX_ERROR_BODY; + while !body.is_char_boundary(end) { + end -= 1; + } + std::borrow::Cow::Owned(format!("{}…", &body[..end])) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use arr_core::Language; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + use super::{DeepL, DeepLConfig}; + use crate::translate::{Backend, Batch, BatchCue, Error}; + + const KEY: &str = "test-key-not-really-fx"; + + fn deepl(base_url: String) -> DeepL { + DeepL::with_base_url( + DeepLConfig { + auth_key: KEY.to_owned(), + }, + base_url, + ) + .expect("client builds") + } + + fn batch(source: Language, target: Language, texts: &[&str]) -> Batch { + Batch { + source, + target, + cues: texts + .iter() + .enumerate() + .map(|(index, text)| BatchCue { + number: index + 1, + text: (*text).to_owned(), + }) + .collect(), + } + } + + fn reply_body(translations: &[&str]) -> String { + let items: Vec = translations + .iter() + .map(|text| format!(r#"{{"detected_source_language":"EN","text":{text}}}"#)) + .collect(); + format!(r#"{{"translations":[{}]}}"#, items.join(",")) + } + + async fn mount_translate(server: &MockServer, status: u16, body: &str) { + Mock::given(method("POST")) + .and(path("/translate")) + .respond_with(ResponseTemplate::new(status).set_body_string(body)) + .mount(server) + .await; + } + + #[test] + fn a_free_tier_key_selects_the_free_host() { + let free = DeepL::new(DeepLConfig { + auth_key: "abc:fx".to_owned(), + }) + .expect("client builds"); + assert!( + free.base_url + .as_str() + .starts_with("https://api-free.deepl.com"), + "free key must pick the free host" + ); + + let pro = DeepL::new(DeepLConfig { + auth_key: "abc123".to_owned(), + }) + .expect("client builds"); + assert!(pro.base_url.as_str().starts_with("https://api.deepl.com")); + } + + #[test] + fn pt_pt_and_pt_br_are_distinct_targets_and_bare_pt_is_not_one() { + let client = deepl("http://localhost".to_owned()); + assert!(client.supports(&Language::PortuguesePortugal)); + assert!(client.supports(&Language::PortugueseBrazil)); + assert!(client.supports(&Language::Other("en".to_owned()))); + assert!(!client.supports(&Language::Other("pt".to_owned()))); + assert!(!client.supports(&Language::PortugueseUnverified)); + } + + #[tokio::test] + async fn a_batch_translates_with_numbers_and_target_preserved() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/translate")) + .and(header("DeepL-Auth-Key", KEY)) + .respond_with( + ResponseTemplate::new(200).set_body_string(reply_body(&[r#""Olá""#, r#""Mundo""#])), + ) + .mount(&server) + .await; + let client = deepl(server.uri()); + + let reply = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["Hello", "World"], + )) + .await + .expect("the mock answers"); + + assert_eq!(reply.len(), 2); + assert_eq!(reply[0].number, 1); + assert_eq!(reply[0].text, "Olá"); + assert_eq!(reply[1].number, 2); + + let request = &server.received_requests().await.expect("requests recorded")[0]; + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); + assert_eq!(body["target_lang"], "PT-PT"); + assert_eq!(body["source_lang"], "EN"); + assert_eq!(body["text"][1], "World"); + } + + #[tokio::test] + async fn bare_portuguese_travels_as_source_but_never_as_target() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await; + let client = deepl(server.uri()); + + client + .translate(&batch( + Language::Other("pt".to_owned()), + Language::PortugueseBrazil, + &["Olá"], + )) + .await + .expect("bare pt is fine as a source"); + + let request = &server.received_requests().await.expect("requests recorded")[0]; + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); + assert_eq!(body["source_lang"], "PT"); + } + + #[tokio::test] + async fn characters_sent_are_reported_as_billed() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""x""#])).await; + let client = deepl(server.uri()); + assert_eq!(client.characters_billed(), 0); + + client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["four"], + )) + .await + .expect("the mock answers"); + + assert_eq!(client.characters_billed(), 4); + } + + #[tokio::test] + async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""um""#])).await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["one", "two"], + )) + .await + .expect_err("two texts in, one out must fail"); + + assert!(matches!(error, Error::CueMismatch { .. })); + } + + #[tokio::test] + async fn a_bad_key_is_unauthorized_and_not_worth_retrying() { + let server = MockServer::start().await; + mount_translate(&server, 403, "").await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("403 must be refused"); + + assert!(matches!(error, Error::Unauthorized { .. })); + assert!(!error.is_transient()); + } + + #[tokio::test] + async fn a_spent_quota_is_the_queue_state_not_a_failure() { + let server = MockServer::start().await; + mount_translate(&server, super::STATUS_QUOTA_EXCEEDED, "").await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("a spent quota must surface as RateLimited"); + + assert!(matches!( + error, + Error::RateLimited { + retry_after: None, + .. + } + )); + assert!(error.is_transient()); + } + + #[tokio::test] + async fn a_rate_limit_honours_retry_after() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/translate")) + .respond_with( + ResponseTemplate::new(429) + .set_body_string("") + .insert_header("Retry-After", "30"), + ) + .mount(&server) + .await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("429 must surface as RateLimited"); + + let Error::RateLimited { retry_after, .. } = error else { + panic!("expected RateLimited, got {error:?}") + }; + assert_eq!(retry_after, Some(Duration::from_secs(30))); + } +} diff --git a/crates/arr-subs/src/lib.rs b/crates/arr-subs/src/lib.rs index 4aedaba..25ccc6f 100644 --- a/crates/arr-subs/src/lib.rs +++ b/crates/arr-subs/src/lib.rs @@ -12,8 +12,10 @@ //! //! Translation backends each sit behind their own cargo feature — //! `translate-openai`, `translate-deepl`, `translate-google`, -//! `translate-command`. The features are declared; the backends themselves are -//! later issues (#191, #192, #193). What they share — the [`translate::Backend`] +//! `translate-command`. The features are declared; the `openai` and +//! `command` backends are later issues (#191, #193), while DeepL +//! (`deepl`) and Google Translate (`google`) live here behind +//! their flags. What they share — the [`translate::Backend`] //! trait, cue chunking, reply validation — lives in [`translate`], and the SRT //! cues it works in live in [`srt`]. Which compiled-in backend runs is a //! database setting, so switching engines never needs a rebuild. @@ -21,6 +23,10 @@ use std::{fmt, future::Future, pin::Pin}; pub mod decode; +#[cfg(feature = "translate-deepl")] +pub mod deepl; +#[cfg(feature = "translate-google")] +pub mod google; // `unused_crate_dependencies` is a per-target lint; `wiremock` is only used // from the `tests/podnapisi.rs` integration target, not from this library // target's own `#[cfg(test)]` module. @@ -34,7 +40,11 @@ pub mod podnapisi; pub mod srt; pub mod translate; +#[cfg(feature = "translate-deepl")] +pub use deepl::{DeepL, DeepLConfig}; pub use error::{Error, Result}; +#[cfg(feature = "translate-google")] +pub use google::{Google, GoogleConfig}; pub use model::{ Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat, }; @@ -209,10 +219,10 @@ mod tests { } #[test] - fn no_engine_is_compiled_in_by_default() { - // The default feature set is empty (Cargo.toml), matching #198's - // scope: the backends themselves are later issues. - assert!(super::compiled_engines().is_empty()); + fn every_compiled_engine_is_a_known_engine_name() { + // Which engines are compiled in depends on the feature flags; what + // must hold under every flag combination is that nothing appears in + // `compiled_engines` that the settings API would not recognise. for engine in super::compiled_engines() { assert!(super::ENGINES.contains(&engine)); } From 3e5fdc6908aa387dba38c9997cec14461dfdc741 Mon Sep 17 00:00:00 2001 From: Miguel Palhas Date: Tue, 25 Aug 2026 01:01:17 +0100 Subject: [PATCH 3/3] feat(arr): translate through Google Translate --- crates/arr-subs/src/google.rs | 618 ++++++++++++++++++++++++++++++++++ crates/arr-subs/src/lib.rs | 2 +- 2 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 crates/arr-subs/src/google.rs diff --git a/crates/arr-subs/src/google.rs b/crates/arr-subs/src/google.rs new file mode 100644 index 0000000..88c84ca --- /dev/null +++ b/crates/arr-subs/src/google.rs @@ -0,0 +1,618 @@ +//! The Google Translate translation backend (`DESIGN.md` §15, issue #192). +//! +//! The cheap, ubiquitous option — and the reason #190's contract insists a +//! backend own its language support: Google Translate v2's `pt` target is +//! Brazilian in practice and it has no pt-PT at all. This module says so in +//! [`Backend::supports`] rather than handing back Brazilian text under a +//! pt-PT request. +//! +//! One request translates a whole batch: the texts go out as a `q` array and +//! come back as a same-length `translations` array, order preserved, so cue +//! numbering never leaves [`crate::translate`]. Two quirks of the API are +//! handled here so callers never see them: +//! +//! * replies HTML-escape characters even with `format: "text"` +//! (`"`, `'`), so they are decoded before returning; +//! * per-character usage is not echoed back, so characters sent are counted +//! on this side and reported for #197's budget. +//! +//! The API key arrives from bootstrap config or environment per §10 — +//! `ARR_TRANSLATE_GOOGLE_API_KEY`, plumbed through `arr-daemon`'s config — +//! never a database row. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use reqwest::{Client, StatusCode, Url}; +use serde::{Deserialize, Serialize}; + +use arr_core::Language; + +use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue}; +use crate::{Error as SubsError, ProviderId, Result}; + +/// The public Google Cloud Translation v2 endpoint. +pub const DEFAULT_BASE_URL: &str = "https://translation.googleapis.com/language/translate/v2"; + +/// How much of an unexpected response body is worth keeping in an error. +const MAX_ERROR_BODY: usize = 512; + +/// Credentials for Google Translate (DESIGN.md §10). +#[derive(Clone, PartialEq, Eq)] +pub struct GoogleConfig { + /// The Cloud Translation API key, sent as the `key` query parameter. + pub api_key: String, +} + +// Hand-written: a derived Debug would print the key. +impl std::fmt::Debug for GoogleConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GoogleConfig") + .field("api_key", &"") + .finish() + } +} + +/// The Google Translate client behind the [`Backend`] trait. +pub struct Google { + http: Client, + base_url: Url, + config: GoogleConfig, + id: BackendId, + billed: AtomicU64, +} + +// Hand-written: the key must never reach a log line. +impl std::fmt::Debug for Google { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Google") + .field("base_url", &self.base_url.as_str()) + .finish_non_exhaustive() + } +} + +impl Google { + /// A client against the real Google Translate API. + /// + /// # Errors + /// + /// When the HTTP client cannot be built or the default base URL does not + /// parse. + pub fn new(config: GoogleConfig) -> Result { + Self::with_base_url(config, DEFAULT_BASE_URL) + } + + /// A client against an arbitrary v2-compatible endpoint. Tests point this + /// at a local mock; an operator with a proxy can do the same. + /// + /// # Errors + /// + /// When the HTTP client cannot be built or `base_url` does not parse. + pub fn with_base_url(config: GoogleConfig, base_url: impl AsRef) -> Result { + let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config { + provider: ProviderId::new("google"), + detail: error.to_string(), + })?; + Ok(Self { + http: Client::new(), + base_url: url, + config, + id: BackendId::new("google"), + billed: AtomicU64::new(0), + }) + } +} + +/// The code Google's `target` field expects, or `None` when Google cannot +/// express the language honestly. +/// +/// `pt` is deliberately passed through bare: that is Google's one Portuguese +/// target, and it is Brazilian. pt-PT and por-unverified refuse here. +#[must_use] +fn google_code(language: &Language) -> Option { + match language { + // Google's `pt` is Brazilian in practice; saying "pt-PT" to it would + // still return Brazilian text under our name. + Language::PortuguesePortugal | Language::PortugueseUnverified => None, + Language::PortugueseBrazil => Some("pt".to_owned()), + Language::Other(tag) => other_tag(tag), + } +} + +/// The `source` hint. Both Portuguese variants hint plain `pt`: a source +/// describes text arr already holds, it promises nothing about output — and +/// what output variant Google produces is exactly what [`google_code`] +/// refuses to fake. +#[must_use] +fn google_source_code(language: &Language) -> Option { + match language { + Language::PortuguesePortugal | Language::PortugueseBrazil => Some("pt".to_owned()), + Language::PortugueseUnverified => None, + Language::Other(tag) => other_tag(tag), + } +} + +#[must_use] +fn other_tag(tag: &str) -> Option { + let tag = tag.trim(); + let mut parts = tag.split('-'); + let primary = parts.next()?; + if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic()) + { + return None; + } + let mut out = primary.to_ascii_lowercase(); + for part in parts { + if part.is_empty() + || !part + .chars() + .all(|c: char| c.is_ascii_alphanumeric() || c == '_') + { + return None; + } + out.push('-'); + out.push_str(part); + } + Some(out) +} + +#[derive(Serialize)] +struct Request<'a> { + q: &'a [String], + #[serde(skip_serializing_if = "Option::is_none")] + source: Option<&'a str>, + target: &'a str, + format: &'static str, +} + +#[derive(Deserialize)] +struct Response { + data: ResponseData, +} + +#[derive(Deserialize)] +struct ResponseData { + translations: Vec, +} + +#[derive(Deserialize)] +struct Translation { + #[serde(rename = "translatedText")] + translated_text: String, +} + +#[derive(Deserialize)] +struct ApiError { + error: ApiErrorBody, +} + +#[derive(Deserialize)] +struct ApiErrorBody { + code: i64, + message: String, +} + +impl Backend for Google { + fn id(&self) -> BackendId { + self.id.clone() + } + + fn supports(&self, target: &Language) -> bool { + google_code(target).is_some() + } + + fn characters_billed(&self) -> u64 { + self.billed.load(Ordering::Relaxed) + } + + fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> { + Box::pin(async move { + let texts: Vec = batch.cues.iter().map(|cue| cue.text.clone()).collect(); + + // Unreachable when called through `translate`, which checks + // `supports` first; kept honest so a direct caller still refuses. + let Some(target) = google_code(&batch.target) else { + return Err(Error::UnsupportedTarget { + backend: self.id(), + language: batch.target.clone(), + }); + }; + let source = google_source_code(&batch.source); + + let mut url = self.base_url.clone(); + url.query_pairs_mut() + .append_pair("key", &self.config.api_key); + + let reply = self + .http + .post(url) + .json(&Request { + q: &texts, + source: source.as_deref(), + target: &target, + format: "text", + }) + .send() + .await + .map_err(|error| Error::Transport { + backend: self.id(), + source: Box::new(error), + })?; + + let status = reply.status(); + match status { + StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }), + StatusCode::TOO_MANY_REQUESTS => { + return Err(Error::RateLimited { + backend: self.id(), + retry_after: None, + }); + } + _ => {} + } + + let bytes = reply.bytes().await.map_err(|error| Error::Transport { + backend: self.id(), + source: Box::new(error), + })?; + if !status.is_success() { + let detail = api_error_detail(status, &bytes); + if status.is_server_error() { + return Err(Error::Transport { + backend: self.id(), + source: detail.into(), + }); + } + return Err(Error::Malformed { + backend: self.id(), + detail, + }); + } + + let parsed: Response = + serde_json::from_slice(&bytes).map_err(|error| Error::Malformed { + backend: self.id(), + detail: error.to_string(), + })?; + + if parsed.data.translations.len() != texts.len() { + return Err(Error::CueMismatch { + backend: self.id(), + detail: format!( + "sent {} texts, got {} translations back", + texts.len(), + parsed.data.translations.len() + ), + }); + } + + self.billed.fetch_add( + texts.iter().map(|text| text.chars().count() as u64).sum(), + Ordering::Relaxed, + ); + + Ok(parsed + .data + .translations + .into_iter() + .zip(&batch.cues) + .map(|(translation, cue)| TranslatedCue { + number: cue.number, + // The API HTML-escapes even plain-text replies. + text: unescape_html(&translation.translated_text), + }) + .collect()) + }) + } +} + +fn api_error_detail(status: StatusCode, body: &[u8]) -> String { + if let Ok(parsed) = serde_json::from_slice::(body) { + return format!("HTTP {}: {}", parsed.error.code, parsed.error.message); + } + let lossy = String::from_utf8_lossy(body); + let mut end = MAX_ERROR_BODY.min(lossy.len()); + while !lossy.is_char_boundary(end) { + end -= 1; + } + format!("HTTP {status}: {}", &lossy[..end]) +} + +/// Decode the HTML entities Google's v2 replies carry even in text mode. +/// +/// Named entities appear for quotes and ampersands (`"`, `'`, +/// `&`) and numeric ones for anything non-ASCII the encoder feels like +/// escaping. Leaving them in would write visible `'` into every subtitle +/// line containing an apostrophe. +#[must_use] +pub fn unescape_html(text: &str) -> String { + if !text.contains('&') { + return text.to_owned(); + } + + let mut out = String::with_capacity(text.len()); + let mut rest = text; + + while let Some(position) = rest.find('&') { + out.push_str(&rest[..position]); + let tail = &rest[position..]; + let semicolon = match tail.find(';') { + Some(index) if index <= 8 => index, + _ => { + out.push('&'); + rest = &tail[1..]; + continue; + } + }; + let entity = &tail[1..semicolon]; + if let Some(character) = named_entity(entity).or_else(|| numeric_entity(entity)) { + out.push(character); + rest = &tail[semicolon + 1..]; + } else { + out.push('&'); + rest = &tail[1..]; + } + } + out.push_str(rest); + out +} + +fn named_entity(entity: &str) -> Option { + match entity { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + _ => None, + } +} + +fn numeric_entity(entity: &str) -> Option { + let code = if let Some(hex) = entity.strip_prefix("#x").or(entity.strip_prefix("#X")) { + u32::from_str_radix(hex, 16).ok()? + } else { + entity.strip_prefix('#')?.parse::().ok()? + }; + char::from_u32(code) +} + +#[cfg(test)] +mod tests { + use arr_core::Language; + use wiremock::matchers::method; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + use super::{Google, GoogleConfig}; + use crate::translate::{Backend, Batch, BatchCue, Error}; + + const KEY: &str = "test-api-key"; + + fn google(base_url: String) -> Google { + Google::with_base_url( + GoogleConfig { + api_key: KEY.to_owned(), + }, + base_url, + ) + .expect("client builds") + } + + fn batch(source: Language, target: Language, texts: &[&str]) -> Batch { + Batch { + source, + target, + cues: texts + .iter() + .enumerate() + .map(|(index, text)| BatchCue { + number: index + 1, + text: (*text).to_owned(), + }) + .collect(), + } + } + + fn reply_body(translations: &[&str]) -> String { + let items: Vec = translations + .iter() + .map(|text| format!(r#"{{"translatedText":{text}}}"#)) + .collect(); + format!( + r#"{{"data":{{"translations":[{items}]}}}}"#, + items = items.join(",") + ) + } + + async fn mount_translate(server: &MockServer, status: u16, body: &str) { + // The base URL *is* the endpoint under test, so only the method + // discriminates here. + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status).set_body_string(body)) + .mount(server) + .await; + } + + #[test] + fn pt_is_brazilian_so_pt_pt_refuses_rather_than_pretends() { + let client = google("http://localhost".to_owned()); + assert!(client.supports(&Language::PortugueseBrazil)); + assert!(client.supports(&Language::Other("pt".to_owned()))); + assert!(client.supports(&Language::Other("en".to_owned()))); + assert!(!client.supports(&Language::PortuguesePortugal)); + assert!(!client.supports(&Language::PortugueseUnverified)); + } + + #[tokio::test] + async fn an_unsupported_target_never_reaches_the_wire() { + let server = MockServer::start().await; + let client = google(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("pt-PT must be refused outright"); + + assert!(matches!(error, Error::UnsupportedTarget { .. })); + assert!(server + .received_requests() + .await + .expect("requests") + .is_empty()); + } + + #[tokio::test] + async fn a_batch_translates_with_numbers_preserved_and_entities_decoded() { + let server = MockServer::start().await; + mount_translate( + &server, + 200, + &reply_body(&[r#""Ol\u00e1 "mundo"!""#]), + ) + .await; + let client = google(server.uri()); + + let reply = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["Hello world!"], + )) + .await + .expect("the mock answers"); + + assert_eq!(reply.len(), 1); + assert_eq!(reply[0].number, 1); + assert_eq!(reply[0].text, "Olá \"mundo\"!"); + + let request = &server.received_requests().await.expect("requests recorded")[0]; + assert_eq!( + request + .url + .query_pairs() + .find(|(key, _)| key == "key") + .map(|(_, value)| value), + Some(std::borrow::Cow::Borrowed(KEY)) + ); + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); + assert_eq!(body["target"], "pt"); + assert_eq!(body["source"], "en"); + assert_eq!(body["format"], "text"); + } + + #[tokio::test] + async fn a_portuguese_source_hints_pt_without_claiming_a_variant() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await; + let client = google(server.uri()); + + client + .translate(&batch( + Language::PortuguesePortugal, + Language::Other("en".to_owned()), + &["Olá"], + )) + .await + .expect("pt-PT is a fine source"); + + let request = &server.received_requests().await.expect("requests recorded")[0]; + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); + assert_eq!(body["source"], "pt"); + } + + #[tokio::test] + async fn characters_sent_are_reported_as_billed() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""x""#])).await; + let client = google(server.uri()); + assert_eq!(client.characters_billed(), 0); + + client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["four"], + )) + .await + .expect("the mock answers"); + + assert_eq!(client.characters_billed(), 4); + } + + #[tokio::test] + async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""um""#])).await; + let client = google(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["one", "two"], + )) + .await + .expect_err("two texts in, one out must fail"); + + assert!(matches!(error, Error::CueMismatch { .. })); + } + + #[tokio::test] + async fn a_bad_key_is_unauthorized_and_not_worth_retrying() { + let server = MockServer::start().await; + mount_translate( + &server, + 403, + r#"{"error":{"code":403,"message":"API key not valid"}}"#, + ) + .await; + let client = google(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["hi"], + )) + .await + .expect_err("403 must be refused"); + + assert!(matches!(error, Error::Unauthorized { .. })); + assert!(!error.is_transient()); + } + + #[tokio::test] + async fn a_rate_limit_surfaces_as_the_queue_state() { + let server = MockServer::start().await; + mount_translate(&server, 429, "").await; + let client = google(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["hi"], + )) + .await + .expect_err("429 must surface as RateLimited"); + + assert!(matches!(error, Error::RateLimited { .. })); + assert!(error.is_transient()); + } + + #[test] + fn html_entities_decode_including_numeric_ones() { + use super::unescape_html; + assert_eq!(unescape_html("plain"), "plain"); + assert_eq!(unescape_html("a & b"), "a & b"); + assert_eq!(unescape_html(""x""), "\"x\""); + assert_eq!(unescape_html("n'then"), "n'then"); + assert_eq!(unescape_html("café"), "café"); + assert_eq!(unescape_html("café"), "café"); + // A stray ampersand that is not an entity stays one. + assert_eq!(unescape_html("fish & chips"), "fish & chips"); + } +} diff --git a/crates/arr-subs/src/lib.rs b/crates/arr-subs/src/lib.rs index 25ccc6f..ff0604e 100644 --- a/crates/arr-subs/src/lib.rs +++ b/crates/arr-subs/src/lib.rs @@ -13,7 +13,7 @@ //! Translation backends each sit behind their own cargo feature — //! `translate-openai`, `translate-deepl`, `translate-google`, //! `translate-command`. The features are declared; the `openai` and -//! `command` backends are later issues (#191, #193), while DeepL +//! `command` backends are later issues (#191, #193), while `DeepL` //! (`deepl`) and Google Translate (`google`) live here behind //! their flags. What they share — the [`translate::Backend`] //! trait, cue chunking, reply validation — lives in [`translate`], and the SRT