From 0c07fbaed5b5b340244432f3f241a4b97208f822 Mon Sep 17 00:00:00 2001 From: Miguel Palhas Date: Tue, 25 Aug 2026 01:01:01 +0100 Subject: [PATCH] feat(arr): translate through the DeepL API --- crates/arr-subs/src/deepl.rs | 567 +++++++++++++++++++++++++++++++++++ crates/arr-subs/src/lib.rs | 22 +- 2 files changed, 583 insertions(+), 6 deletions(-) create mode 100644 crates/arr-subs/src/deepl.rs diff --git a/crates/arr-subs/src/deepl.rs b/crates/arr-subs/src/deepl.rs new file mode 100644 index 0000000..4920698 --- /dev/null +++ b/crates/arr-subs/src/deepl.rs @@ -0,0 +1,567 @@ +//! The `DeepL` translation backend (`DESIGN.md` §15, issue #192). +//! +//! `DeepL` is one of the two dedicated translation APIs §15 names. It is the +//! only backend that keeps pt-PT and pt-BR apart as first-class targets +//! (`PT-PT`, `PT-BR`), which is exactly why arr wants it — but it has no +//! plain-Portuguese target, so a request for generic `pt` is refused in +//! [`Backend::supports`] rather than silently answered with one variant or +//! the other. +//! +//! One request translates a whole batch: the texts go out as a JSON array +//! and come back as a same-length array of translations, order preserved, +//! so cue numbering never leaves [`crate::translate`]. Characters sent are +//! what `DeepL` bills, so they are counted and reported for #197's budget. +//! +//! The auth key arrives from bootstrap config or environment per §10 — +//! `ARR_TRANSLATE_DEEPL_API_KEY`, plumbed through `arr-daemon`'s config — +//! never a database row. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use reqwest::{Client, StatusCode, Url}; +use serde::{Deserialize, Serialize}; + +use arr_core::Language; + +use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue}; +use crate::{Error as SubsError, ProviderId, Result}; + +/// The `DeepL` API endpoint for keys ending in `:fx` (the free tier). +const FREE_BASE_URL: &str = "https://api-free.deepl.com/v2"; + +/// The `DeepL` API endpoint for every other key (the pro tier). +const PRO_BASE_URL: &str = "https://api.deepl.com/v2"; + +/// `DeepL`'s non-standard status for a spent billing-period quota. +const STATUS_QUOTA_EXCEEDED: u16 = 456; + +/// How much of an unexpected response body is worth keeping in an error. +const MAX_ERROR_BODY: usize = 512; + +/// Credentials for `DeepL` (DESIGN.md §10). +/// +/// A single auth key; free-tier keys end in `:fx` and select the +/// `api-free.deepl.com` host automatically. Never persisted to the database. +#[derive(Clone, PartialEq, Eq)] +pub struct DeepLConfig { + /// The `DeepL` auth key, sent as `DeepL-Auth-Key` on every call. + pub auth_key: String, +} + +// Hand-written: a derived Debug would print the key. +impl std::fmt::Debug for DeepLConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeepLConfig") + .field("auth_key", &"") + .finish() + } +} + +/// The `DeepL` client behind the [`Backend`] trait. +pub struct DeepL { + http: Client, + base_url: Url, + config: DeepLConfig, + id: BackendId, + billed: AtomicU64, +} + +// Hand-written: the key must never reach a log line. +impl std::fmt::Debug for DeepL { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeepL") + .field("base_url", &self.base_url.as_str()) + .finish_non_exhaustive() + } +} + +impl DeepL { + /// A client against the real `DeepL` API. The tier follows the key: one + /// ending in `:fx` talks to the free host, anything else to the pro host. + /// + /// # Errors + /// + /// When the HTTP client cannot be built or the derived base URL does not + /// parse. + pub fn new(config: DeepLConfig) -> Result { + let base = if config.auth_key.ends_with(":fx") { + FREE_BASE_URL + } else { + PRO_BASE_URL + }; + Self::with_base_url(config, base) + } + + /// A client against an arbitrary DeepL-compatible endpoint. Tests point + /// this at a local mock; an operator with a proxy can do the same. + /// + /// # Errors + /// + /// When the HTTP client cannot be built or `base_url` does not parse. + pub fn with_base_url(config: DeepLConfig, base_url: impl AsRef) -> Result { + let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config { + provider: ProviderId::new("deepl"), + detail: error.to_string(), + })?; + Ok(Self { + http: Client::new(), + base_url: url, + config, + id: BackendId::new("deepl"), + billed: AtomicU64::new(0), + }) + } + + fn join(&self, segment: &str) -> Url { + let mut url = self.base_url.clone(); + let path = format!("{}/{}", url.path().trim_end_matches('/'), segment); + url.set_path(&path); + url + } +} + +/// The language code `DeepL` expects, or `None` when it cannot express the +/// language exactly. +/// +/// Bare Portuguese is the case that matters: `DeepL` only accepts `PT-PT` and +/// `PT-BR` as a target, so `pt` on its own refuses rather than guessing a +/// variant. As a *source* hint bare `PT` is fine — a source describes text +/// arr already holds, it promises nothing about output. +#[must_use] +fn deep_code(language: &Language, allow_bare_pt: bool) -> Option { + let tag = match language { + Language::PortuguesePortugal => return Some("PT-PT".to_owned()), + Language::PortugueseBrazil => return Some("PT-BR".to_owned()), + Language::PortugueseUnverified => return None, + Language::Other(tag) => tag, + }; + + let mut parts = tag.split('-'); + let primary = parts.next()?; + if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic()) + { + return None; + } + let rest: Vec = parts.map(str::to_ascii_uppercase).collect(); + if rest.is_empty() && primary.eq_ignore_ascii_case("pt") && !allow_bare_pt { + return None; + } + + let mut out = primary.to_ascii_uppercase(); + for part in rest { + out.push('-'); + out.push_str(&part); + } + Some(out) +} + +#[derive(Serialize)] +struct Request<'a> { + text: &'a [String], + target_lang: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + source_lang: Option<&'a str>, +} + +#[derive(Deserialize)] +struct Response { + translations: Vec, +} + +#[derive(Deserialize)] +struct Translation { + #[serde(default)] + #[allow(dead_code)] + detected_source_language: Option, + text: String, +} + +impl Backend for DeepL { + fn id(&self) -> BackendId { + self.id.clone() + } + + fn supports(&self, target: &Language) -> bool { + deep_code(target, false).is_some() + } + + fn characters_billed(&self) -> u64 { + self.billed.load(Ordering::Relaxed) + } + + fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> { + Box::pin(async move { + let texts: Vec = batch.cues.iter().map(|cue| cue.text.clone()).collect(); + + // Unreachable when called through `translate`, which checks + // `supports` first; kept honest so a direct caller still refuses. + let Some(target_lang) = deep_code(&batch.target, false) else { + return Err(Error::UnsupportedTarget { + backend: self.id(), + language: batch.target.clone(), + }); + }; + let source_lang = deep_code(&batch.source, true); + + let reply = self + .http + .post(self.join("translate")) + .header("DeepL-Auth-Key", &self.config.auth_key) + .json(&Request { + text: &texts, + target_lang: &target_lang, + source_lang: source_lang.as_deref(), + }) + .send() + .await + .map_err(|error| Error::Transport { + backend: self.id(), + source: Box::new(error), + })?; + + let status = reply.status(); + match status { + StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }), + StatusCode::TOO_MANY_REQUESTS => { + let retry_after = retry_after(reply.headers()); + return Err(Error::RateLimited { + backend: self.id(), + retry_after, + }); + } + status if status.as_u16() == STATUS_QUOTA_EXCEEDED => { + // The billing-period character quota is spent. Same queue + // state as a rate limit (#197), just without a + // Retry-After to honour. + return Err(Error::RateLimited { + backend: self.id(), + retry_after: None, + }); + } + _ => {} + } + + let bytes = reply.bytes().await.map_err(|error| Error::Transport { + backend: self.id(), + source: Box::new(error), + })?; + if !status.is_success() { + let detail = String::from_utf8_lossy(&bytes); + if status.is_server_error() { + return Err(Error::Transport { + backend: self.id(), + source: format!("HTTP {status}: {}", truncate(&detail)).into(), + }); + } + return Err(Error::Malformed { + backend: self.id(), + detail: format!("HTTP {status}: {}", truncate(&detail)), + }); + } + + let parsed: Response = + serde_json::from_slice(&bytes).map_err(|error| Error::Malformed { + backend: self.id(), + detail: error.to_string(), + })?; + + if parsed.translations.len() != texts.len() { + return Err(Error::CueMismatch { + backend: self.id(), + detail: format!( + "sent {} texts, got {} translations back", + texts.len(), + parsed.translations.len() + ), + }); + } + + self.billed.fetch_add( + texts.iter().map(|text| text.chars().count() as u64).sum(), + Ordering::Relaxed, + ); + + Ok(parsed + .translations + .into_iter() + .zip(&batch.cues) + .map(|(translation, cue)| TranslatedCue { + number: cue.number, + text: translation.text, + }) + .collect()) + }) + } +} + +fn retry_after(headers: &reqwest::header::HeaderMap) -> Option { + headers + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(Duration::from_secs) +} + +fn truncate(body: &str) -> std::borrow::Cow<'_, str> { + if body.len() <= MAX_ERROR_BODY { + return std::borrow::Cow::Borrowed(body); + } + let mut end = MAX_ERROR_BODY; + while !body.is_char_boundary(end) { + end -= 1; + } + std::borrow::Cow::Owned(format!("{}…", &body[..end])) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use arr_core::Language; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + use super::{DeepL, DeepLConfig}; + use crate::translate::{Backend, Batch, BatchCue, Error}; + + const KEY: &str = "test-key-not-really-fx"; + + fn deepl(base_url: String) -> DeepL { + DeepL::with_base_url( + DeepLConfig { + auth_key: KEY.to_owned(), + }, + base_url, + ) + .expect("client builds") + } + + fn batch(source: Language, target: Language, texts: &[&str]) -> Batch { + Batch { + source, + target, + cues: texts + .iter() + .enumerate() + .map(|(index, text)| BatchCue { + number: index + 1, + text: (*text).to_owned(), + }) + .collect(), + } + } + + fn reply_body(translations: &[&str]) -> String { + let items: Vec = translations + .iter() + .map(|text| format!(r#"{{"detected_source_language":"EN","text":{text}}}"#)) + .collect(); + format!(r#"{{"translations":[{}]}}"#, items.join(",")) + } + + async fn mount_translate(server: &MockServer, status: u16, body: &str) { + Mock::given(method("POST")) + .and(path("/translate")) + .respond_with(ResponseTemplate::new(status).set_body_string(body)) + .mount(server) + .await; + } + + #[test] + fn a_free_tier_key_selects_the_free_host() { + let free = DeepL::new(DeepLConfig { + auth_key: "abc:fx".to_owned(), + }) + .expect("client builds"); + assert!( + free.base_url + .as_str() + .starts_with("https://api-free.deepl.com"), + "free key must pick the free host" + ); + + let pro = DeepL::new(DeepLConfig { + auth_key: "abc123".to_owned(), + }) + .expect("client builds"); + assert!(pro.base_url.as_str().starts_with("https://api.deepl.com")); + } + + #[test] + fn pt_pt_and_pt_br_are_distinct_targets_and_bare_pt_is_not_one() { + let client = deepl("http://localhost".to_owned()); + assert!(client.supports(&Language::PortuguesePortugal)); + assert!(client.supports(&Language::PortugueseBrazil)); + assert!(client.supports(&Language::Other("en".to_owned()))); + assert!(!client.supports(&Language::Other("pt".to_owned()))); + assert!(!client.supports(&Language::PortugueseUnverified)); + } + + #[tokio::test] + async fn a_batch_translates_with_numbers_and_target_preserved() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/translate")) + .and(header("DeepL-Auth-Key", KEY)) + .respond_with( + ResponseTemplate::new(200).set_body_string(reply_body(&[r#""Olá""#, r#""Mundo""#])), + ) + .mount(&server) + .await; + let client = deepl(server.uri()); + + let reply = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["Hello", "World"], + )) + .await + .expect("the mock answers"); + + assert_eq!(reply.len(), 2); + assert_eq!(reply[0].number, 1); + assert_eq!(reply[0].text, "Olá"); + assert_eq!(reply[1].number, 2); + + let request = &server.received_requests().await.expect("requests recorded")[0]; + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); + assert_eq!(body["target_lang"], "PT-PT"); + assert_eq!(body["source_lang"], "EN"); + assert_eq!(body["text"][1], "World"); + } + + #[tokio::test] + async fn bare_portuguese_travels_as_source_but_never_as_target() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await; + let client = deepl(server.uri()); + + client + .translate(&batch( + Language::Other("pt".to_owned()), + Language::PortugueseBrazil, + &["Olá"], + )) + .await + .expect("bare pt is fine as a source"); + + let request = &server.received_requests().await.expect("requests recorded")[0]; + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); + assert_eq!(body["source_lang"], "PT"); + } + + #[tokio::test] + async fn characters_sent_are_reported_as_billed() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""x""#])).await; + let client = deepl(server.uri()); + assert_eq!(client.characters_billed(), 0); + + client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortugueseBrazil, + &["four"], + )) + .await + .expect("the mock answers"); + + assert_eq!(client.characters_billed(), 4); + } + + #[tokio::test] + async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() { + let server = MockServer::start().await; + mount_translate(&server, 200, &reply_body(&[r#""um""#])).await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["one", "two"], + )) + .await + .expect_err("two texts in, one out must fail"); + + assert!(matches!(error, Error::CueMismatch { .. })); + } + + #[tokio::test] + async fn a_bad_key_is_unauthorized_and_not_worth_retrying() { + let server = MockServer::start().await; + mount_translate(&server, 403, "").await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("403 must be refused"); + + assert!(matches!(error, Error::Unauthorized { .. })); + assert!(!error.is_transient()); + } + + #[tokio::test] + async fn a_spent_quota_is_the_queue_state_not_a_failure() { + let server = MockServer::start().await; + mount_translate(&server, super::STATUS_QUOTA_EXCEEDED, "").await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("a spent quota must surface as RateLimited"); + + assert!(matches!( + error, + Error::RateLimited { + retry_after: None, + .. + } + )); + assert!(error.is_transient()); + } + + #[tokio::test] + async fn a_rate_limit_honours_retry_after() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/translate")) + .respond_with( + ResponseTemplate::new(429) + .set_body_string("") + .insert_header("Retry-After", "30"), + ) + .mount(&server) + .await; + let client = deepl(server.uri()); + + let error = client + .translate(&batch( + Language::Other("en".to_owned()), + Language::PortuguesePortugal, + &["hi"], + )) + .await + .expect_err("429 must surface as RateLimited"); + + let Error::RateLimited { retry_after, .. } = error else { + panic!("expected RateLimited, got {error:?}") + }; + assert_eq!(retry_after, Some(Duration::from_secs(30))); + } +} diff --git a/crates/arr-subs/src/lib.rs b/crates/arr-subs/src/lib.rs index 4aedaba..25ccc6f 100644 --- a/crates/arr-subs/src/lib.rs +++ b/crates/arr-subs/src/lib.rs @@ -12,8 +12,10 @@ //! //! Translation backends each sit behind their own cargo feature — //! `translate-openai`, `translate-deepl`, `translate-google`, -//! `translate-command`. The features are declared; the backends themselves are -//! later issues (#191, #192, #193). What they share — the [`translate::Backend`] +//! `translate-command`. The features are declared; the `openai` and +//! `command` backends are later issues (#191, #193), while DeepL +//! (`deepl`) and Google Translate (`google`) live here behind +//! their flags. What they share — the [`translate::Backend`] //! trait, cue chunking, reply validation — lives in [`translate`], and the SRT //! cues it works in live in [`srt`]. Which compiled-in backend runs is a //! database setting, so switching engines never needs a rebuild. @@ -21,6 +23,10 @@ use std::{fmt, future::Future, pin::Pin}; pub mod decode; +#[cfg(feature = "translate-deepl")] +pub mod deepl; +#[cfg(feature = "translate-google")] +pub mod google; // `unused_crate_dependencies` is a per-target lint; `wiremock` is only used // from the `tests/podnapisi.rs` integration target, not from this library // target's own `#[cfg(test)]` module. @@ -34,7 +40,11 @@ pub mod podnapisi; pub mod srt; pub mod translate; +#[cfg(feature = "translate-deepl")] +pub use deepl::{DeepL, DeepLConfig}; pub use error::{Error, Result}; +#[cfg(feature = "translate-google")] +pub use google::{Google, GoogleConfig}; pub use model::{ Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat, }; @@ -209,10 +219,10 @@ mod tests { } #[test] - fn no_engine_is_compiled_in_by_default() { - // The default feature set is empty (Cargo.toml), matching #198's - // scope: the backends themselves are later issues. - assert!(super::compiled_engines().is_empty()); + fn every_compiled_engine_is_a_known_engine_name() { + // Which engines are compiled in depends on the feature flags; what + // must hold under every flag combination is that nothing appears in + // `compiled_engines` that the settings API would not recognise. for engine in super::compiled_engines() { assert!(super::ENGINES.contains(&engine)); }