feat(arr): translate through the DeepL API

This commit is contained in:
Miguel Palhas
2026-08-25 01:01:01 +01:00
parent ef0fac4933
commit 0c07fbaed5
2 changed files with 583 additions and 6 deletions
+567
View File
@@ -0,0 +1,567 @@
//! The `DeepL` translation backend (`DESIGN.md` §15, issue #192).
//!
//! `DeepL` is one of the two dedicated translation APIs §15 names. It is the
//! only backend that keeps pt-PT and pt-BR apart as first-class targets
//! (`PT-PT`, `PT-BR`), which is exactly why arr wants it — but it has no
//! plain-Portuguese target, so a request for generic `pt` is refused in
//! [`Backend::supports`] rather than silently answered with one variant or
//! the other.
//!
//! One request translates a whole batch: the texts go out as a JSON array
//! and come back as a same-length array of translations, order preserved,
//! so cue numbering never leaves [`crate::translate`]. Characters sent are
//! what `DeepL` bills, so they are counted and reported for #197's budget.
//!
//! The auth key arrives from bootstrap config or environment per §10 —
//! `ARR_TRANSLATE_DEEPL_API_KEY`, plumbed through `arr-daemon`'s config —
//! never a database row.
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use reqwest::{Client, StatusCode, Url};
use serde::{Deserialize, Serialize};
use arr_core::Language;
use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue};
use crate::{Error as SubsError, ProviderId, Result};
/// The `DeepL` API endpoint for keys ending in `:fx` (the free tier).
const FREE_BASE_URL: &str = "https://api-free.deepl.com/v2";
/// The `DeepL` API endpoint for every other key (the pro tier).
const PRO_BASE_URL: &str = "https://api.deepl.com/v2";
/// `DeepL`'s non-standard status for a spent billing-period quota.
const STATUS_QUOTA_EXCEEDED: u16 = 456;
/// How much of an unexpected response body is worth keeping in an error.
const MAX_ERROR_BODY: usize = 512;
/// Credentials for `DeepL` (DESIGN.md §10).
///
/// A single auth key; free-tier keys end in `:fx` and select the
/// `api-free.deepl.com` host automatically. Never persisted to the database.
#[derive(Clone, PartialEq, Eq)]
pub struct DeepLConfig {
/// The `DeepL` auth key, sent as `DeepL-Auth-Key` on every call.
pub auth_key: String,
}
// Hand-written: a derived Debug would print the key.
impl std::fmt::Debug for DeepLConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeepLConfig")
.field("auth_key", &"<redacted>")
.finish()
}
}
/// The `DeepL` client behind the [`Backend`] trait.
pub struct DeepL {
http: Client,
base_url: Url,
config: DeepLConfig,
id: BackendId,
billed: AtomicU64,
}
// Hand-written: the key must never reach a log line.
impl std::fmt::Debug for DeepL {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeepL")
.field("base_url", &self.base_url.as_str())
.finish_non_exhaustive()
}
}
impl DeepL {
/// A client against the real `DeepL` API. The tier follows the key: one
/// ending in `:fx` talks to the free host, anything else to the pro host.
///
/// # Errors
///
/// When the HTTP client cannot be built or the derived base URL does not
/// parse.
pub fn new(config: DeepLConfig) -> Result<Self> {
let base = if config.auth_key.ends_with(":fx") {
FREE_BASE_URL
} else {
PRO_BASE_URL
};
Self::with_base_url(config, base)
}
/// A client against an arbitrary DeepL-compatible endpoint. Tests point
/// this at a local mock; an operator with a proxy can do the same.
///
/// # Errors
///
/// When the HTTP client cannot be built or `base_url` does not parse.
pub fn with_base_url(config: DeepLConfig, base_url: impl AsRef<str>) -> Result<Self> {
let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config {
provider: ProviderId::new("deepl"),
detail: error.to_string(),
})?;
Ok(Self {
http: Client::new(),
base_url: url,
config,
id: BackendId::new("deepl"),
billed: AtomicU64::new(0),
})
}
fn join(&self, segment: &str) -> Url {
let mut url = self.base_url.clone();
let path = format!("{}/{}", url.path().trim_end_matches('/'), segment);
url.set_path(&path);
url
}
}
/// The language code `DeepL` expects, or `None` when it cannot express the
/// language exactly.
///
/// Bare Portuguese is the case that matters: `DeepL` only accepts `PT-PT` and
/// `PT-BR` as a target, so `pt` on its own refuses rather than guessing a
/// variant. As a *source* hint bare `PT` is fine — a source describes text
/// arr already holds, it promises nothing about output.
#[must_use]
fn deep_code(language: &Language, allow_bare_pt: bool) -> Option<String> {
let tag = match language {
Language::PortuguesePortugal => return Some("PT-PT".to_owned()),
Language::PortugueseBrazil => return Some("PT-BR".to_owned()),
Language::PortugueseUnverified => return None,
Language::Other(tag) => tag,
};
let mut parts = tag.split('-');
let primary = parts.next()?;
if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic())
{
return None;
}
let rest: Vec<String> = parts.map(str::to_ascii_uppercase).collect();
if rest.is_empty() && primary.eq_ignore_ascii_case("pt") && !allow_bare_pt {
return None;
}
let mut out = primary.to_ascii_uppercase();
for part in rest {
out.push('-');
out.push_str(&part);
}
Some(out)
}
#[derive(Serialize)]
struct Request<'a> {
text: &'a [String],
target_lang: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
source_lang: Option<&'a str>,
}
#[derive(Deserialize)]
struct Response {
translations: Vec<Translation>,
}
#[derive(Deserialize)]
struct Translation {
#[serde(default)]
#[allow(dead_code)]
detected_source_language: Option<String>,
text: String,
}
impl Backend for DeepL {
fn id(&self) -> BackendId {
self.id.clone()
}
fn supports(&self, target: &Language) -> bool {
deep_code(target, false).is_some()
}
fn characters_billed(&self) -> u64 {
self.billed.load(Ordering::Relaxed)
}
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
Box::pin(async move {
let texts: Vec<String> = batch.cues.iter().map(|cue| cue.text.clone()).collect();
// Unreachable when called through `translate`, which checks
// `supports` first; kept honest so a direct caller still refuses.
let Some(target_lang) = deep_code(&batch.target, false) else {
return Err(Error::UnsupportedTarget {
backend: self.id(),
language: batch.target.clone(),
});
};
let source_lang = deep_code(&batch.source, true);
let reply = self
.http
.post(self.join("translate"))
.header("DeepL-Auth-Key", &self.config.auth_key)
.json(&Request {
text: &texts,
target_lang: &target_lang,
source_lang: source_lang.as_deref(),
})
.send()
.await
.map_err(|error| Error::Transport {
backend: self.id(),
source: Box::new(error),
})?;
let status = reply.status();
match status {
StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }),
StatusCode::TOO_MANY_REQUESTS => {
let retry_after = retry_after(reply.headers());
return Err(Error::RateLimited {
backend: self.id(),
retry_after,
});
}
status if status.as_u16() == STATUS_QUOTA_EXCEEDED => {
// The billing-period character quota is spent. Same queue
// state as a rate limit (#197), just without a
// Retry-After to honour.
return Err(Error::RateLimited {
backend: self.id(),
retry_after: None,
});
}
_ => {}
}
let bytes = reply.bytes().await.map_err(|error| Error::Transport {
backend: self.id(),
source: Box::new(error),
})?;
if !status.is_success() {
let detail = String::from_utf8_lossy(&bytes);
if status.is_server_error() {
return Err(Error::Transport {
backend: self.id(),
source: format!("HTTP {status}: {}", truncate(&detail)).into(),
});
}
return Err(Error::Malformed {
backend: self.id(),
detail: format!("HTTP {status}: {}", truncate(&detail)),
});
}
let parsed: Response =
serde_json::from_slice(&bytes).map_err(|error| Error::Malformed {
backend: self.id(),
detail: error.to_string(),
})?;
if parsed.translations.len() != texts.len() {
return Err(Error::CueMismatch {
backend: self.id(),
detail: format!(
"sent {} texts, got {} translations back",
texts.len(),
parsed.translations.len()
),
});
}
self.billed.fetch_add(
texts.iter().map(|text| text.chars().count() as u64).sum(),
Ordering::Relaxed,
);
Ok(parsed
.translations
.into_iter()
.zip(&batch.cues)
.map(|(translation, cue)| TranslatedCue {
number: cue.number,
text: translation.text,
})
.collect())
})
}
}
fn retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
headers
.get(reqwest::header::RETRY_AFTER)?
.to_str()
.ok()?
.trim()
.parse::<u64>()
.ok()
.map(Duration::from_secs)
}
fn truncate(body: &str) -> std::borrow::Cow<'_, str> {
if body.len() <= MAX_ERROR_BODY {
return std::borrow::Cow::Borrowed(body);
}
let mut end = MAX_ERROR_BODY;
while !body.is_char_boundary(end) {
end -= 1;
}
std::borrow::Cow::Owned(format!("{}…", &body[..end]))
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use arr_core::Language;
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use super::{DeepL, DeepLConfig};
use crate::translate::{Backend, Batch, BatchCue, Error};
const KEY: &str = "test-key-not-really-fx";
fn deepl(base_url: String) -> DeepL {
DeepL::with_base_url(
DeepLConfig {
auth_key: KEY.to_owned(),
},
base_url,
)
.expect("client builds")
}
fn batch(source: Language, target: Language, texts: &[&str]) -> Batch {
Batch {
source,
target,
cues: texts
.iter()
.enumerate()
.map(|(index, text)| BatchCue {
number: index + 1,
text: (*text).to_owned(),
})
.collect(),
}
}
fn reply_body(translations: &[&str]) -> String {
let items: Vec<String> = translations
.iter()
.map(|text| format!(r#"{{"detected_source_language":"EN","text":{text}}}"#))
.collect();
format!(r#"{{"translations":[{}]}}"#, items.join(","))
}
async fn mount_translate(server: &MockServer, status: u16, body: &str) {
Mock::given(method("POST"))
.and(path("/translate"))
.respond_with(ResponseTemplate::new(status).set_body_string(body))
.mount(server)
.await;
}
#[test]
fn a_free_tier_key_selects_the_free_host() {
let free = DeepL::new(DeepLConfig {
auth_key: "abc:fx".to_owned(),
})
.expect("client builds");
assert!(
free.base_url
.as_str()
.starts_with("https://api-free.deepl.com"),
"free key must pick the free host"
);
let pro = DeepL::new(DeepLConfig {
auth_key: "abc123".to_owned(),
})
.expect("client builds");
assert!(pro.base_url.as_str().starts_with("https://api.deepl.com"));
}
#[test]
fn pt_pt_and_pt_br_are_distinct_targets_and_bare_pt_is_not_one() {
let client = deepl("http://localhost".to_owned());
assert!(client.supports(&Language::PortuguesePortugal));
assert!(client.supports(&Language::PortugueseBrazil));
assert!(client.supports(&Language::Other("en".to_owned())));
assert!(!client.supports(&Language::Other("pt".to_owned())));
assert!(!client.supports(&Language::PortugueseUnverified));
}
#[tokio::test]
async fn a_batch_translates_with_numbers_and_target_preserved() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/translate"))
.and(header("DeepL-Auth-Key", KEY))
.respond_with(
ResponseTemplate::new(200).set_body_string(reply_body(&[r#""Olá""#, r#""Mundo""#])),
)
.mount(&server)
.await;
let client = deepl(server.uri());
let reply = client
.translate(&batch(
Language::Other("en".to_owned()),
Language::PortuguesePortugal,
&["Hello", "World"],
))
.await
.expect("the mock answers");
assert_eq!(reply.len(), 2);
assert_eq!(reply[0].number, 1);
assert_eq!(reply[0].text, "Olá");
assert_eq!(reply[1].number, 2);
let request = &server.received_requests().await.expect("requests recorded")[0];
let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body");
assert_eq!(body["target_lang"], "PT-PT");
assert_eq!(body["source_lang"], "EN");
assert_eq!(body["text"][1], "World");
}
#[tokio::test]
async fn bare_portuguese_travels_as_source_but_never_as_target() {
let server = MockServer::start().await;
mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await;
let client = deepl(server.uri());
client
.translate(&batch(
Language::Other("pt".to_owned()),
Language::PortugueseBrazil,
&["Olá"],
))
.await
.expect("bare pt is fine as a source");
let request = &server.received_requests().await.expect("requests recorded")[0];
let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body");
assert_eq!(body["source_lang"], "PT");
}
#[tokio::test]
async fn characters_sent_are_reported_as_billed() {
let server = MockServer::start().await;
mount_translate(&server, 200, &reply_body(&[r#""x""#])).await;
let client = deepl(server.uri());
assert_eq!(client.characters_billed(), 0);
client
.translate(&batch(
Language::Other("en".to_owned()),
Language::PortugueseBrazil,
&["four"],
))
.await
.expect("the mock answers");
assert_eq!(client.characters_billed(), 4);
}
#[tokio::test]
async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() {
let server = MockServer::start().await;
mount_translate(&server, 200, &reply_body(&[r#""um""#])).await;
let client = deepl(server.uri());
let error = client
.translate(&batch(
Language::Other("en".to_owned()),
Language::PortuguesePortugal,
&["one", "two"],
))
.await
.expect_err("two texts in, one out must fail");
assert!(matches!(error, Error::CueMismatch { .. }));
}
#[tokio::test]
async fn a_bad_key_is_unauthorized_and_not_worth_retrying() {
let server = MockServer::start().await;
mount_translate(&server, 403, "").await;
let client = deepl(server.uri());
let error = client
.translate(&batch(
Language::Other("en".to_owned()),
Language::PortuguesePortugal,
&["hi"],
))
.await
.expect_err("403 must be refused");
assert!(matches!(error, Error::Unauthorized { .. }));
assert!(!error.is_transient());
}
#[tokio::test]
async fn a_spent_quota_is_the_queue_state_not_a_failure() {
let server = MockServer::start().await;
mount_translate(&server, super::STATUS_QUOTA_EXCEEDED, "").await;
let client = deepl(server.uri());
let error = client
.translate(&batch(
Language::Other("en".to_owned()),
Language::PortuguesePortugal,
&["hi"],
))
.await
.expect_err("a spent quota must surface as RateLimited");
assert!(matches!(
error,
Error::RateLimited {
retry_after: None,
..
}
));
assert!(error.is_transient());
}
#[tokio::test]
async fn a_rate_limit_honours_retry_after() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/translate"))
.respond_with(
ResponseTemplate::new(429)
.set_body_string("")
.insert_header("Retry-After", "30"),
)
.mount(&server)
.await;
let client = deepl(server.uri());
let error = client
.translate(&batch(
Language::Other("en".to_owned()),
Language::PortuguesePortugal,
&["hi"],
))
.await
.expect_err("429 must surface as RateLimited");
let Error::RateLimited { retry_after, .. } = error else {
panic!("expected RateLimited, got {error:?}")
};
assert_eq!(retry_after, Some(Duration::from_secs(30)));
}
}
+16 -6
View File
@@ -12,8 +12,10 @@
//!
//! Translation backends each sit behind their own cargo feature —
//! `translate-openai`, `translate-deepl`, `translate-google`,
//! `translate-command`. The features are declared; the backends themselves are
//! later issues (#191, #192, #193). What they share — the [`translate::Backend`]
//! `translate-command`. The features are declared; the `openai` and
//! `command` backends are later issues (#191, #193), while DeepL
//! (`deepl`) and Google Translate (`google`) live here behind
//! their flags. What they share — the [`translate::Backend`]
//! trait, cue chunking, reply validation — lives in [`translate`], and the SRT
//! cues it works in live in [`srt`]. Which compiled-in backend runs is a
//! database setting, so switching engines never needs a rebuild.
@@ -21,6 +23,10 @@
use std::{fmt, future::Future, pin::Pin};
pub mod decode;
#[cfg(feature = "translate-deepl")]
pub mod deepl;
#[cfg(feature = "translate-google")]
pub mod google;
// `unused_crate_dependencies` is a per-target lint; `wiremock` is only used
// from the `tests/podnapisi.rs` integration target, not from this library
// target's own `#[cfg(test)]` module.
@@ -34,7 +40,11 @@ pub mod podnapisi;
pub mod srt;
pub mod translate;
#[cfg(feature = "translate-deepl")]
pub use deepl::{DeepL, DeepLConfig};
pub use error::{Error, Result};
#[cfg(feature = "translate-google")]
pub use google::{Google, GoogleConfig};
pub use model::{
Candidate, CandidateId, Fetched, MediaFile, MediaRef, ProviderId, SearchRequest, SubtitleFormat,
};
@@ -209,10 +219,10 @@ mod tests {
}
#[test]
fn no_engine_is_compiled_in_by_default() {
// The default feature set is empty (Cargo.toml), matching #198's
// scope: the backends themselves are later issues.
assert!(super::compiled_engines().is_empty());
fn every_compiled_engine_is_a_known_engine_name() {
// Which engines are compiled in depends on the feature flags; what
// must hold under every flag combination is that nothing appears in
// `compiled_engines` that the settings API would not recognise.
for engine in super::compiled_engines() {
assert!(super::ENGINES.contains(&engine));
}