568 lines
19 KiB
Rust
568 lines
19 KiB
Rust
//! The `DeepL` translation backend (`DESIGN.md` §15, issue #192).
|
|
//!
|
|
//! `DeepL` is one of the two dedicated translation APIs §15 names. It is the
|
|
//! only backend that keeps pt-PT and pt-BR apart as first-class targets
|
|
//! (`PT-PT`, `PT-BR`), which is exactly why arr wants it — but it has no
|
|
//! plain-Portuguese target, so a request for generic `pt` is refused in
|
|
//! [`Backend::supports`] rather than silently answered with one variant or
|
|
//! the other.
|
|
//!
|
|
//! One request translates a whole batch: the texts go out as a JSON array
|
|
//! and come back as a same-length array of translations, order preserved,
|
|
//! so cue numbering never leaves [`crate::translate`]. Characters sent are
|
|
//! what `DeepL` bills, so they are counted and reported for #197's budget.
|
|
//!
|
|
//! The auth key arrives from bootstrap config or environment per §10 —
|
|
//! `ARR_TRANSLATE_DEEPL_API_KEY`, plumbed through `arr-daemon`'s config —
|
|
//! never a database row.
|
|
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
use std::time::Duration;
|
|
|
|
use reqwest::{Client, StatusCode, Url};
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use arr_core::Language;
|
|
|
|
use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue};
|
|
use crate::{Error as SubsError, ProviderId, Result};
|
|
|
|
/// The `DeepL` API endpoint for keys ending in `:fx` (the free tier).
|
|
const FREE_BASE_URL: &str = "https://api-free.deepl.com/v2";
|
|
|
|
/// The `DeepL` API endpoint for every other key (the pro tier).
|
|
const PRO_BASE_URL: &str = "https://api.deepl.com/v2";
|
|
|
|
/// `DeepL`'s non-standard status for a spent billing-period quota.
|
|
const STATUS_QUOTA_EXCEEDED: u16 = 456;
|
|
|
|
/// How much of an unexpected response body is worth keeping in an error.
|
|
const MAX_ERROR_BODY: usize = 512;
|
|
|
|
/// Credentials for `DeepL` (DESIGN.md §10).
|
|
///
|
|
/// A single auth key; free-tier keys end in `:fx` and select the
|
|
/// `api-free.deepl.com` host automatically. Never persisted to the database.
|
|
#[derive(Clone, PartialEq, Eq)]
|
|
pub struct DeepLConfig {
|
|
/// The `DeepL` auth key, sent as `DeepL-Auth-Key` on every call.
|
|
pub auth_key: String,
|
|
}
|
|
|
|
// Hand-written: a derived Debug would print the key.
|
|
impl std::fmt::Debug for DeepLConfig {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("DeepLConfig")
|
|
.field("auth_key", &"<redacted>")
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
/// The `DeepL` client behind the [`Backend`] trait.
|
|
pub struct DeepL {
|
|
http: Client,
|
|
base_url: Url,
|
|
config: DeepLConfig,
|
|
id: BackendId,
|
|
billed: AtomicU64,
|
|
}
|
|
|
|
// Hand-written: the key must never reach a log line.
|
|
impl std::fmt::Debug for DeepL {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("DeepL")
|
|
.field("base_url", &self.base_url.as_str())
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl DeepL {
|
|
/// A client against the real `DeepL` API. The tier follows the key: one
|
|
/// ending in `:fx` talks to the free host, anything else to the pro host.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// When the HTTP client cannot be built or the derived base URL does not
|
|
/// parse.
|
|
pub fn new(config: DeepLConfig) -> Result<Self> {
|
|
let base = if config.auth_key.ends_with(":fx") {
|
|
FREE_BASE_URL
|
|
} else {
|
|
PRO_BASE_URL
|
|
};
|
|
Self::with_base_url(config, base)
|
|
}
|
|
|
|
/// A client against an arbitrary DeepL-compatible endpoint. Tests point
|
|
/// this at a local mock; an operator with a proxy can do the same.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// When the HTTP client cannot be built or `base_url` does not parse.
|
|
pub fn with_base_url(config: DeepLConfig, base_url: impl AsRef<str>) -> Result<Self> {
|
|
let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config {
|
|
provider: ProviderId::new("deepl"),
|
|
detail: error.to_string(),
|
|
})?;
|
|
Ok(Self {
|
|
http: Client::new(),
|
|
base_url: url,
|
|
config,
|
|
id: BackendId::new("deepl"),
|
|
billed: AtomicU64::new(0),
|
|
})
|
|
}
|
|
|
|
fn join(&self, segment: &str) -> Url {
|
|
let mut url = self.base_url.clone();
|
|
let path = format!("{}/{}", url.path().trim_end_matches('/'), segment);
|
|
url.set_path(&path);
|
|
url
|
|
}
|
|
}
|
|
|
|
/// The language code `DeepL` expects, or `None` when it cannot express the
|
|
/// language exactly.
|
|
///
|
|
/// Bare Portuguese is the case that matters: `DeepL` only accepts `PT-PT` and
|
|
/// `PT-BR` as a target, so `pt` on its own refuses rather than guessing a
|
|
/// variant. As a *source* hint bare `PT` is fine — a source describes text
|
|
/// arr already holds, it promises nothing about output.
|
|
#[must_use]
|
|
fn deep_code(language: &Language, allow_bare_pt: bool) -> Option<String> {
|
|
let tag = match language {
|
|
Language::PortuguesePortugal => return Some("PT-PT".to_owned()),
|
|
Language::PortugueseBrazil => return Some("PT-BR".to_owned()),
|
|
Language::PortugueseUnverified => return None,
|
|
Language::Other(tag) => tag,
|
|
};
|
|
|
|
let mut parts = tag.split('-');
|
|
let primary = parts.next()?;
|
|
if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic())
|
|
{
|
|
return None;
|
|
}
|
|
let rest: Vec<String> = parts.map(str::to_ascii_uppercase).collect();
|
|
if rest.is_empty() && primary.eq_ignore_ascii_case("pt") && !allow_bare_pt {
|
|
return None;
|
|
}
|
|
|
|
let mut out = primary.to_ascii_uppercase();
|
|
for part in rest {
|
|
out.push('-');
|
|
out.push_str(&part);
|
|
}
|
|
Some(out)
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct Request<'a> {
|
|
text: &'a [String],
|
|
target_lang: &'a str,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
source_lang: Option<&'a str>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct Response {
|
|
translations: Vec<Translation>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct Translation {
|
|
#[serde(default)]
|
|
#[allow(dead_code)]
|
|
detected_source_language: Option<String>,
|
|
text: String,
|
|
}
|
|
|
|
impl Backend for DeepL {
|
|
fn id(&self) -> BackendId {
|
|
self.id.clone()
|
|
}
|
|
|
|
fn supports(&self, target: &Language) -> bool {
|
|
deep_code(target, false).is_some()
|
|
}
|
|
|
|
fn characters_billed(&self) -> u64 {
|
|
self.billed.load(Ordering::Relaxed)
|
|
}
|
|
|
|
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
|
|
Box::pin(async move {
|
|
let texts: Vec<String> = batch.cues.iter().map(|cue| cue.text.clone()).collect();
|
|
|
|
// Unreachable when called through `translate`, which checks
|
|
// `supports` first; kept honest so a direct caller still refuses.
|
|
let Some(target_lang) = deep_code(&batch.target, false) else {
|
|
return Err(Error::UnsupportedTarget {
|
|
backend: self.id(),
|
|
language: batch.target.clone(),
|
|
});
|
|
};
|
|
let source_lang = deep_code(&batch.source, true);
|
|
|
|
let reply = self
|
|
.http
|
|
.post(self.join("translate"))
|
|
.header("DeepL-Auth-Key", &self.config.auth_key)
|
|
.json(&Request {
|
|
text: &texts,
|
|
target_lang: &target_lang,
|
|
source_lang: source_lang.as_deref(),
|
|
})
|
|
.send()
|
|
.await
|
|
.map_err(|error| Error::Transport {
|
|
backend: self.id(),
|
|
source: Box::new(error),
|
|
})?;
|
|
|
|
let status = reply.status();
|
|
match status {
|
|
StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }),
|
|
StatusCode::TOO_MANY_REQUESTS => {
|
|
let retry_after = retry_after(reply.headers());
|
|
return Err(Error::RateLimited {
|
|
backend: self.id(),
|
|
retry_after,
|
|
});
|
|
}
|
|
status if status.as_u16() == STATUS_QUOTA_EXCEEDED => {
|
|
// The billing-period character quota is spent. Same queue
|
|
// state as a rate limit (#197), just without a
|
|
// Retry-After to honour.
|
|
return Err(Error::RateLimited {
|
|
backend: self.id(),
|
|
retry_after: None,
|
|
});
|
|
}
|
|
_ => {}
|
|
}
|
|
|
|
let bytes = reply.bytes().await.map_err(|error| Error::Transport {
|
|
backend: self.id(),
|
|
source: Box::new(error),
|
|
})?;
|
|
if !status.is_success() {
|
|
let detail = String::from_utf8_lossy(&bytes);
|
|
if status.is_server_error() {
|
|
return Err(Error::Transport {
|
|
backend: self.id(),
|
|
source: format!("HTTP {status}: {}", truncate(&detail)).into(),
|
|
});
|
|
}
|
|
return Err(Error::Malformed {
|
|
backend: self.id(),
|
|
detail: format!("HTTP {status}: {}", truncate(&detail)),
|
|
});
|
|
}
|
|
|
|
let parsed: Response =
|
|
serde_json::from_slice(&bytes).map_err(|error| Error::Malformed {
|
|
backend: self.id(),
|
|
detail: error.to_string(),
|
|
})?;
|
|
|
|
if parsed.translations.len() != texts.len() {
|
|
return Err(Error::CueMismatch {
|
|
backend: self.id(),
|
|
detail: format!(
|
|
"sent {} texts, got {} translations back",
|
|
texts.len(),
|
|
parsed.translations.len()
|
|
),
|
|
});
|
|
}
|
|
|
|
self.billed.fetch_add(
|
|
texts.iter().map(|text| text.chars().count() as u64).sum(),
|
|
Ordering::Relaxed,
|
|
);
|
|
|
|
Ok(parsed
|
|
.translations
|
|
.into_iter()
|
|
.zip(&batch.cues)
|
|
.map(|(translation, cue)| TranslatedCue {
|
|
number: cue.number,
|
|
text: translation.text,
|
|
})
|
|
.collect())
|
|
})
|
|
}
|
|
}
|
|
|
|
fn retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
|
|
headers
|
|
.get(reqwest::header::RETRY_AFTER)?
|
|
.to_str()
|
|
.ok()?
|
|
.trim()
|
|
.parse::<u64>()
|
|
.ok()
|
|
.map(Duration::from_secs)
|
|
}
|
|
|
|
fn truncate(body: &str) -> std::borrow::Cow<'_, str> {
|
|
if body.len() <= MAX_ERROR_BODY {
|
|
return std::borrow::Cow::Borrowed(body);
|
|
}
|
|
let mut end = MAX_ERROR_BODY;
|
|
while !body.is_char_boundary(end) {
|
|
end -= 1;
|
|
}
|
|
std::borrow::Cow::Owned(format!("{}…", &body[..end]))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::time::Duration;
|
|
|
|
use arr_core::Language;
|
|
use wiremock::matchers::{header, method, path};
|
|
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
|
|
use super::{DeepL, DeepLConfig};
|
|
use crate::translate::{Backend, Batch, BatchCue, Error};
|
|
|
|
const KEY: &str = "test-key-not-really-fx";
|
|
|
|
fn deepl(base_url: String) -> DeepL {
|
|
DeepL::with_base_url(
|
|
DeepLConfig {
|
|
auth_key: KEY.to_owned(),
|
|
},
|
|
base_url,
|
|
)
|
|
.expect("client builds")
|
|
}
|
|
|
|
fn batch(source: Language, target: Language, texts: &[&str]) -> Batch {
|
|
Batch {
|
|
source,
|
|
target,
|
|
cues: texts
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(index, text)| BatchCue {
|
|
number: index + 1,
|
|
text: (*text).to_owned(),
|
|
})
|
|
.collect(),
|
|
}
|
|
}
|
|
|
|
fn reply_body(translations: &[&str]) -> String {
|
|
let items: Vec<String> = translations
|
|
.iter()
|
|
.map(|text| format!(r#"{{"detected_source_language":"EN","text":{text}}}"#))
|
|
.collect();
|
|
format!(r#"{{"translations":[{}]}}"#, items.join(","))
|
|
}
|
|
|
|
async fn mount_translate(server: &MockServer, status: u16, body: &str) {
|
|
Mock::given(method("POST"))
|
|
.and(path("/translate"))
|
|
.respond_with(ResponseTemplate::new(status).set_body_string(body))
|
|
.mount(server)
|
|
.await;
|
|
}
|
|
|
|
#[test]
|
|
fn a_free_tier_key_selects_the_free_host() {
|
|
let free = DeepL::new(DeepLConfig {
|
|
auth_key: "abc:fx".to_owned(),
|
|
})
|
|
.expect("client builds");
|
|
assert!(
|
|
free.base_url
|
|
.as_str()
|
|
.starts_with("https://api-free.deepl.com"),
|
|
"free key must pick the free host"
|
|
);
|
|
|
|
let pro = DeepL::new(DeepLConfig {
|
|
auth_key: "abc123".to_owned(),
|
|
})
|
|
.expect("client builds");
|
|
assert!(pro.base_url.as_str().starts_with("https://api.deepl.com"));
|
|
}
|
|
|
|
#[test]
|
|
fn pt_pt_and_pt_br_are_distinct_targets_and_bare_pt_is_not_one() {
|
|
let client = deepl("http://localhost".to_owned());
|
|
assert!(client.supports(&Language::PortuguesePortugal));
|
|
assert!(client.supports(&Language::PortugueseBrazil));
|
|
assert!(client.supports(&Language::Other("en".to_owned())));
|
|
assert!(!client.supports(&Language::Other("pt".to_owned())));
|
|
assert!(!client.supports(&Language::PortugueseUnverified));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_batch_translates_with_numbers_and_target_preserved() {
|
|
let server = MockServer::start().await;
|
|
Mock::given(method("POST"))
|
|
.and(path("/translate"))
|
|
.and(header("DeepL-Auth-Key", KEY))
|
|
.respond_with(
|
|
ResponseTemplate::new(200).set_body_string(reply_body(&[r#""Olá""#, r#""Mundo""#])),
|
|
)
|
|
.mount(&server)
|
|
.await;
|
|
let client = deepl(server.uri());
|
|
|
|
let reply = client
|
|
.translate(&batch(
|
|
Language::Other("en".to_owned()),
|
|
Language::PortuguesePortugal,
|
|
&["Hello", "World"],
|
|
))
|
|
.await
|
|
.expect("the mock answers");
|
|
|
|
assert_eq!(reply.len(), 2);
|
|
assert_eq!(reply[0].number, 1);
|
|
assert_eq!(reply[0].text, "Olá");
|
|
assert_eq!(reply[1].number, 2);
|
|
|
|
let request = &server.received_requests().await.expect("requests recorded")[0];
|
|
let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body");
|
|
assert_eq!(body["target_lang"], "PT-PT");
|
|
assert_eq!(body["source_lang"], "EN");
|
|
assert_eq!(body["text"][1], "World");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn bare_portuguese_travels_as_source_but_never_as_target() {
|
|
let server = MockServer::start().await;
|
|
mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await;
|
|
let client = deepl(server.uri());
|
|
|
|
client
|
|
.translate(&batch(
|
|
Language::Other("pt".to_owned()),
|
|
Language::PortugueseBrazil,
|
|
&["Olá"],
|
|
))
|
|
.await
|
|
.expect("bare pt is fine as a source");
|
|
|
|
let request = &server.received_requests().await.expect("requests recorded")[0];
|
|
let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body");
|
|
assert_eq!(body["source_lang"], "PT");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn characters_sent_are_reported_as_billed() {
|
|
let server = MockServer::start().await;
|
|
mount_translate(&server, 200, &reply_body(&[r#""x""#])).await;
|
|
let client = deepl(server.uri());
|
|
assert_eq!(client.characters_billed(), 0);
|
|
|
|
client
|
|
.translate(&batch(
|
|
Language::Other("en".to_owned()),
|
|
Language::PortugueseBrazil,
|
|
&["four"],
|
|
))
|
|
.await
|
|
.expect("the mock answers");
|
|
|
|
assert_eq!(client.characters_billed(), 4);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() {
|
|
let server = MockServer::start().await;
|
|
mount_translate(&server, 200, &reply_body(&[r#""um""#])).await;
|
|
let client = deepl(server.uri());
|
|
|
|
let error = client
|
|
.translate(&batch(
|
|
Language::Other("en".to_owned()),
|
|
Language::PortuguesePortugal,
|
|
&["one", "two"],
|
|
))
|
|
.await
|
|
.expect_err("two texts in, one out must fail");
|
|
|
|
assert!(matches!(error, Error::CueMismatch { .. }));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_bad_key_is_unauthorized_and_not_worth_retrying() {
|
|
let server = MockServer::start().await;
|
|
mount_translate(&server, 403, "").await;
|
|
let client = deepl(server.uri());
|
|
|
|
let error = client
|
|
.translate(&batch(
|
|
Language::Other("en".to_owned()),
|
|
Language::PortuguesePortugal,
|
|
&["hi"],
|
|
))
|
|
.await
|
|
.expect_err("403 must be refused");
|
|
|
|
assert!(matches!(error, Error::Unauthorized { .. }));
|
|
assert!(!error.is_transient());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_spent_quota_is_the_queue_state_not_a_failure() {
|
|
let server = MockServer::start().await;
|
|
mount_translate(&server, super::STATUS_QUOTA_EXCEEDED, "").await;
|
|
let client = deepl(server.uri());
|
|
|
|
let error = client
|
|
.translate(&batch(
|
|
Language::Other("en".to_owned()),
|
|
Language::PortuguesePortugal,
|
|
&["hi"],
|
|
))
|
|
.await
|
|
.expect_err("a spent quota must surface as RateLimited");
|
|
|
|
assert!(matches!(
|
|
error,
|
|
Error::RateLimited {
|
|
retry_after: None,
|
|
..
|
|
}
|
|
));
|
|
assert!(error.is_transient());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_rate_limit_honours_retry_after() {
|
|
let server = MockServer::start().await;
|
|
Mock::given(method("POST"))
|
|
.and(path("/translate"))
|
|
.respond_with(
|
|
ResponseTemplate::new(429)
|
|
.set_body_string("")
|
|
.insert_header("Retry-After", "30"),
|
|
)
|
|
.mount(&server)
|
|
.await;
|
|
let client = deepl(server.uri());
|
|
|
|
let error = client
|
|
.translate(&batch(
|
|
Language::Other("en".to_owned()),
|
|
Language::PortuguesePortugal,
|
|
&["hi"],
|
|
))
|
|
.await
|
|
.expect_err("429 must surface as RateLimited");
|
|
|
|
let Error::RateLimited { retry_after, .. } = error else {
|
|
panic!("expected RateLimited, got {error:?}")
|
|
};
|
|
assert_eq!(retry_after, Some(Duration::from_secs(30)));
|
|
}
|
|
}
|