//! The `DeepL` translation backend (`DESIGN.md` §15, issue #192). //! //! `DeepL` is one of the two dedicated translation APIs §15 names. It is the //! only backend that keeps pt-PT and pt-BR apart as first-class targets //! (`PT-PT`, `PT-BR`), which is exactly why arr wants it — but it has no //! plain-Portuguese target, so a request for generic `pt` is refused in //! [`Backend::supports`] rather than silently answered with one variant or //! the other. //! //! One request translates a whole batch: the texts go out as a JSON array //! and come back as a same-length array of translations, order preserved, //! so cue numbering never leaves [`crate::translate`]. Characters sent are //! what `DeepL` bills, so they are counted and reported for #197's budget. //! //! The auth key arrives from bootstrap config or environment per §10 — //! `ARR_TRANSLATE_DEEPL_API_KEY`, plumbed through `arr-daemon`'s config — //! never a database row. use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use reqwest::{Client, StatusCode, Url}; use serde::{Deserialize, Serialize}; use arr_core::Language; use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue}; use crate::{Error as SubsError, ProviderId, Result}; /// The `DeepL` API endpoint for keys ending in `:fx` (the free tier). const FREE_BASE_URL: &str = "https://api-free.deepl.com/v2"; /// The `DeepL` API endpoint for every other key (the pro tier). const PRO_BASE_URL: &str = "https://api.deepl.com/v2"; /// `DeepL`'s non-standard status for a spent billing-period quota. const STATUS_QUOTA_EXCEEDED: u16 = 456; /// How much of an unexpected response body is worth keeping in an error. const MAX_ERROR_BODY: usize = 512; /// Credentials for `DeepL` (DESIGN.md §10). /// /// A single auth key; free-tier keys end in `:fx` and select the /// `api-free.deepl.com` host automatically. Never persisted to the database. #[derive(Clone, PartialEq, Eq)] pub struct DeepLConfig { /// The `DeepL` auth key, sent as `DeepL-Auth-Key` on every call. pub auth_key: String, } // Hand-written: a derived Debug would print the key. impl std::fmt::Debug for DeepLConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DeepLConfig") .field("auth_key", &"") .finish() } } /// The `DeepL` client behind the [`Backend`] trait. pub struct DeepL { http: Client, base_url: Url, config: DeepLConfig, id: BackendId, billed: AtomicU64, } // Hand-written: the key must never reach a log line. impl std::fmt::Debug for DeepL { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DeepL") .field("base_url", &self.base_url.as_str()) .finish_non_exhaustive() } } impl DeepL { /// A client against the real `DeepL` API. The tier follows the key: one /// ending in `:fx` talks to the free host, anything else to the pro host. /// /// # Errors /// /// When the HTTP client cannot be built or the derived base URL does not /// parse. pub fn new(config: DeepLConfig) -> Result { let base = if config.auth_key.ends_with(":fx") { FREE_BASE_URL } else { PRO_BASE_URL }; Self::with_base_url(config, base) } /// A client against an arbitrary DeepL-compatible endpoint. Tests point /// this at a local mock; an operator with a proxy can do the same. /// /// # Errors /// /// When the HTTP client cannot be built or `base_url` does not parse. pub fn with_base_url(config: DeepLConfig, base_url: impl AsRef) -> Result { let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config { provider: ProviderId::new("deepl"), detail: error.to_string(), })?; Ok(Self { http: Client::new(), base_url: url, config, id: BackendId::new("deepl"), billed: AtomicU64::new(0), }) } fn join(&self, segment: &str) -> Url { let mut url = self.base_url.clone(); let path = format!("{}/{}", url.path().trim_end_matches('/'), segment); url.set_path(&path); url } } /// The language code `DeepL` expects, or `None` when it cannot express the /// language exactly. /// /// Bare Portuguese is the case that matters: `DeepL` only accepts `PT-PT` and /// `PT-BR` as a target, so `pt` on its own refuses rather than guessing a /// variant. As a *source* hint bare `PT` is fine — a source describes text /// arr already holds, it promises nothing about output. #[must_use] fn deep_code(language: &Language, allow_bare_pt: bool) -> Option { let tag = match language { Language::PortuguesePortugal => return Some("PT-PT".to_owned()), Language::PortugueseBrazil => return Some("PT-BR".to_owned()), Language::PortugueseUnverified => return None, Language::Other(tag) => tag, }; let mut parts = tag.split('-'); let primary = parts.next()?; if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic()) { return None; } let rest: Vec = parts.map(str::to_ascii_uppercase).collect(); if rest.is_empty() && primary.eq_ignore_ascii_case("pt") && !allow_bare_pt { return None; } let mut out = primary.to_ascii_uppercase(); for part in rest { out.push('-'); out.push_str(&part); } Some(out) } #[derive(Serialize)] struct Request<'a> { text: &'a [String], target_lang: &'a str, #[serde(skip_serializing_if = "Option::is_none")] source_lang: Option<&'a str>, } #[derive(Deserialize)] struct Response { translations: Vec, } #[derive(Deserialize)] struct Translation { #[serde(default)] #[allow(dead_code)] detected_source_language: Option, text: String, } impl Backend for DeepL { fn id(&self) -> BackendId { self.id.clone() } fn supports(&self, target: &Language) -> bool { deep_code(target, false).is_some() } fn characters_billed(&self) -> u64 { self.billed.load(Ordering::Relaxed) } fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> { Box::pin(async move { let texts: Vec = batch.cues.iter().map(|cue| cue.text.clone()).collect(); // Unreachable when called through `translate`, which checks // `supports` first; kept honest so a direct caller still refuses. let Some(target_lang) = deep_code(&batch.target, false) else { return Err(Error::UnsupportedTarget { backend: self.id(), language: batch.target.clone(), }); }; let source_lang = deep_code(&batch.source, true); let reply = self .http .post(self.join("translate")) .header("DeepL-Auth-Key", &self.config.auth_key) .json(&Request { text: &texts, target_lang: &target_lang, source_lang: source_lang.as_deref(), }) .send() .await .map_err(|error| Error::Transport { backend: self.id(), source: Box::new(error), })?; let status = reply.status(); match status { StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }), StatusCode::TOO_MANY_REQUESTS => { let retry_after = retry_after(reply.headers()); return Err(Error::RateLimited { backend: self.id(), retry_after, }); } status if status.as_u16() == STATUS_QUOTA_EXCEEDED => { // The billing-period character quota is spent. Same queue // state as a rate limit (#197), just without a // Retry-After to honour. return Err(Error::RateLimited { backend: self.id(), retry_after: None, }); } _ => {} } let bytes = reply.bytes().await.map_err(|error| Error::Transport { backend: self.id(), source: Box::new(error), })?; if !status.is_success() { let detail = String::from_utf8_lossy(&bytes); if status.is_server_error() { return Err(Error::Transport { backend: self.id(), source: format!("HTTP {status}: {}", truncate(&detail)).into(), }); } return Err(Error::Malformed { backend: self.id(), detail: format!("HTTP {status}: {}", truncate(&detail)), }); } let parsed: Response = serde_json::from_slice(&bytes).map_err(|error| Error::Malformed { backend: self.id(), detail: error.to_string(), })?; if parsed.translations.len() != texts.len() { return Err(Error::CueMismatch { backend: self.id(), detail: format!( "sent {} texts, got {} translations back", texts.len(), parsed.translations.len() ), }); } self.billed.fetch_add( texts.iter().map(|text| text.chars().count() as u64).sum(), Ordering::Relaxed, ); Ok(parsed .translations .into_iter() .zip(&batch.cues) .map(|(translation, cue)| TranslatedCue { number: cue.number, text: translation.text, }) .collect()) }) } } fn retry_after(headers: &reqwest::header::HeaderMap) -> Option { headers .get(reqwest::header::RETRY_AFTER)? .to_str() .ok()? .trim() .parse::() .ok() .map(Duration::from_secs) } fn truncate(body: &str) -> std::borrow::Cow<'_, str> { if body.len() <= MAX_ERROR_BODY { return std::borrow::Cow::Borrowed(body); } let mut end = MAX_ERROR_BODY; while !body.is_char_boundary(end) { end -= 1; } std::borrow::Cow::Owned(format!("{}…", &body[..end])) } #[cfg(test)] mod tests { use std::time::Duration; use arr_core::Language; use wiremock::matchers::{header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; use super::{DeepL, DeepLConfig}; use crate::translate::{Backend, Batch, BatchCue, Error}; const KEY: &str = "test-key-not-really-fx"; fn deepl(base_url: String) -> DeepL { DeepL::with_base_url( DeepLConfig { auth_key: KEY.to_owned(), }, base_url, ) .expect("client builds") } fn batch(source: Language, target: Language, texts: &[&str]) -> Batch { Batch { source, target, cues: texts .iter() .enumerate() .map(|(index, text)| BatchCue { number: index + 1, text: (*text).to_owned(), }) .collect(), } } fn reply_body(translations: &[&str]) -> String { let items: Vec = translations .iter() .map(|text| format!(r#"{{"detected_source_language":"EN","text":{text}}}"#)) .collect(); format!(r#"{{"translations":[{}]}}"#, items.join(",")) } async fn mount_translate(server: &MockServer, status: u16, body: &str) { Mock::given(method("POST")) .and(path("/translate")) .respond_with(ResponseTemplate::new(status).set_body_string(body)) .mount(server) .await; } #[test] fn a_free_tier_key_selects_the_free_host() { let free = DeepL::new(DeepLConfig { auth_key: "abc:fx".to_owned(), }) .expect("client builds"); assert!( free.base_url .as_str() .starts_with("https://api-free.deepl.com"), "free key must pick the free host" ); let pro = DeepL::new(DeepLConfig { auth_key: "abc123".to_owned(), }) .expect("client builds"); assert!(pro.base_url.as_str().starts_with("https://api.deepl.com")); } #[test] fn pt_pt_and_pt_br_are_distinct_targets_and_bare_pt_is_not_one() { let client = deepl("http://localhost".to_owned()); assert!(client.supports(&Language::PortuguesePortugal)); assert!(client.supports(&Language::PortugueseBrazil)); assert!(client.supports(&Language::Other("en".to_owned()))); assert!(!client.supports(&Language::Other("pt".to_owned()))); assert!(!client.supports(&Language::PortugueseUnverified)); } #[tokio::test] async fn a_batch_translates_with_numbers_and_target_preserved() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/translate")) .and(header("DeepL-Auth-Key", KEY)) .respond_with( ResponseTemplate::new(200).set_body_string(reply_body(&[r#""Olá""#, r#""Mundo""#])), ) .mount(&server) .await; let client = deepl(server.uri()); let reply = client .translate(&batch( Language::Other("en".to_owned()), Language::PortuguesePortugal, &["Hello", "World"], )) .await .expect("the mock answers"); assert_eq!(reply.len(), 2); assert_eq!(reply[0].number, 1); assert_eq!(reply[0].text, "Olá"); assert_eq!(reply[1].number, 2); let request = &server.received_requests().await.expect("requests recorded")[0]; let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); assert_eq!(body["target_lang"], "PT-PT"); assert_eq!(body["source_lang"], "EN"); assert_eq!(body["text"][1], "World"); } #[tokio::test] async fn bare_portuguese_travels_as_source_but_never_as_target() { let server = MockServer::start().await; mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await; let client = deepl(server.uri()); client .translate(&batch( Language::Other("pt".to_owned()), Language::PortugueseBrazil, &["Olá"], )) .await .expect("bare pt is fine as a source"); let request = &server.received_requests().await.expect("requests recorded")[0]; let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); assert_eq!(body["source_lang"], "PT"); } #[tokio::test] async fn characters_sent_are_reported_as_billed() { let server = MockServer::start().await; mount_translate(&server, 200, &reply_body(&[r#""x""#])).await; let client = deepl(server.uri()); assert_eq!(client.characters_billed(), 0); client .translate(&batch( Language::Other("en".to_owned()), Language::PortugueseBrazil, &["four"], )) .await .expect("the mock answers"); assert_eq!(client.characters_billed(), 4); } #[tokio::test] async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() { let server = MockServer::start().await; mount_translate(&server, 200, &reply_body(&[r#""um""#])).await; let client = deepl(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortuguesePortugal, &["one", "two"], )) .await .expect_err("two texts in, one out must fail"); assert!(matches!(error, Error::CueMismatch { .. })); } #[tokio::test] async fn a_bad_key_is_unauthorized_and_not_worth_retrying() { let server = MockServer::start().await; mount_translate(&server, 403, "").await; let client = deepl(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortuguesePortugal, &["hi"], )) .await .expect_err("403 must be refused"); assert!(matches!(error, Error::Unauthorized { .. })); assert!(!error.is_transient()); } #[tokio::test] async fn a_spent_quota_is_the_queue_state_not_a_failure() { let server = MockServer::start().await; mount_translate(&server, super::STATUS_QUOTA_EXCEEDED, "").await; let client = deepl(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortuguesePortugal, &["hi"], )) .await .expect_err("a spent quota must surface as RateLimited"); assert!(matches!( error, Error::RateLimited { retry_after: None, .. } )); assert!(error.is_transient()); } #[tokio::test] async fn a_rate_limit_honours_retry_after() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/translate")) .respond_with( ResponseTemplate::new(429) .set_body_string("") .insert_header("Retry-After", "30"), ) .mount(&server) .await; let client = deepl(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortuguesePortugal, &["hi"], )) .await .expect_err("429 must surface as RateLimited"); let Error::RateLimited { retry_after, .. } = error else { panic!("expected RateLimited, got {error:?}") }; assert_eq!(retry_after, Some(Duration::from_secs(30))); } }