feat(arr): translate through Google Translate

2026-08-25 01:01:17 +01:00
parent 0c07fbaed5
commit 3e5fdc6908
2 changed files with 619 additions and 1 deletions
@@ -0,0 +1,618 @@
+//! The Google Translate translation backend (`DESIGN.md` §15, issue #192).
+//!
+//! The cheap, ubiquitous option — and the reason #190's contract insists a
+//! backend own its language support: Google Translate v2's `pt` target is
+//! Brazilian in practice and it has no pt-PT at all. This module says so in
+//! [`Backend::supports`] rather than handing back Brazilian text under a
+//! pt-PT request.
+//!
+//! One request translates a whole batch: the texts go out as a `q` array and
+//! come back as a same-length `translations` array, order preserved, so cue
+//! numbering never leaves [`crate::translate`]. Two quirks of the API are
+//! handled here so callers never see them:
+//!
+//! * replies HTML-escape characters even with `format: "text"`
+//!   (`&quot;`, `&#39;`), so they are decoded before returning;
+//! * per-character usage is not echoed back, so characters sent are counted
+//!   on this side and reported for #197's budget.
+//!
+//! The API key arrives from bootstrap config or environment per §10 —
+//! `ARR_TRANSLATE_GOOGLE_API_KEY`, plumbed through `arr-daemon`'s config —
+//! never a database row.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use reqwest::{Client, StatusCode, Url};
+use serde::{Deserialize, Serialize};
+
+use arr_core::Language;
+
+use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue};
+use crate::{Error as SubsError, ProviderId, Result};
+
+/// The public Google Cloud Translation v2 endpoint.
+pub const DEFAULT_BASE_URL: &str = "https://translation.googleapis.com/language/translate/v2";
+
+/// How much of an unexpected response body is worth keeping in an error.
+const MAX_ERROR_BODY: usize = 512;
+
+/// Credentials for Google Translate (DESIGN.md §10).
+#[derive(Clone, PartialEq, Eq)]
+pub struct GoogleConfig {
+    /// The Cloud Translation API key, sent as the `key` query parameter.
+    pub api_key: String,
+}
+
+// Hand-written: a derived Debug would print the key.
+impl std::fmt::Debug for GoogleConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GoogleConfig")
+            .field("api_key", &"<redacted>")
+            .finish()
+    }
+}
+
+/// The Google Translate client behind the [`Backend`] trait.
+pub struct Google {
+    http: Client,
+    base_url: Url,
+    config: GoogleConfig,
+    id: BackendId,
+    billed: AtomicU64,
+}
+
+// Hand-written: the key must never reach a log line.
+impl std::fmt::Debug for Google {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Google")
+            .field("base_url", &self.base_url.as_str())
+            .finish_non_exhaustive()
+    }
+}
+
+impl Google {
+    /// A client against the real Google Translate API.
+    ///
+    /// # Errors
+    ///
+    /// When the HTTP client cannot be built or the default base URL does not
+    /// parse.
+    pub fn new(config: GoogleConfig) -> Result<Self> {
+        Self::with_base_url(config, DEFAULT_BASE_URL)
+    }
+
+    /// A client against an arbitrary v2-compatible endpoint. Tests point this
+    /// at a local mock; an operator with a proxy can do the same.
+    ///
+    /// # Errors
+    ///
+    /// When the HTTP client cannot be built or `base_url` does not parse.
+    pub fn with_base_url(config: GoogleConfig, base_url: impl AsRef<str>) -> Result<Self> {
+        let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config {
+            provider: ProviderId::new("google"),
+            detail: error.to_string(),
+        })?;
+        Ok(Self {
+            http: Client::new(),
+            base_url: url,
+            config,
+            id: BackendId::new("google"),
+            billed: AtomicU64::new(0),
+        })
+    }
+}
+
+/// The code Google's `target` field expects, or `None` when Google cannot
+/// express the language honestly.
+///
+/// `pt` is deliberately passed through bare: that is Google's one Portuguese
+/// target, and it is Brazilian. pt-PT and por-unverified refuse here.
+#[must_use]
+fn google_code(language: &Language) -> Option<String> {
+    match language {
+        // Google's `pt` is Brazilian in practice; saying "pt-PT" to it would
+        // still return Brazilian text under our name.
+        Language::PortuguesePortugal | Language::PortugueseUnverified => None,
+        Language::PortugueseBrazil => Some("pt".to_owned()),
+        Language::Other(tag) => other_tag(tag),
+    }
+}
+
+/// The `source` hint. Both Portuguese variants hint plain `pt`: a source
+/// describes text arr already holds, it promises nothing about output — and
+/// what output variant Google produces is exactly what [`google_code`]
+/// refuses to fake.
+#[must_use]
+fn google_source_code(language: &Language) -> Option<String> {
+    match language {
+        Language::PortuguesePortugal | Language::PortugueseBrazil => Some("pt".to_owned()),
+        Language::PortugueseUnverified => None,
+        Language::Other(tag) => other_tag(tag),
+    }
+}
+
+#[must_use]
+fn other_tag(tag: &str) -> Option<String> {
+    let tag = tag.trim();
+    let mut parts = tag.split('-');
+    let primary = parts.next()?;
+    if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic())
+    {
+        return None;
+    }
+    let mut out = primary.to_ascii_lowercase();
+    for part in parts {
+        if part.is_empty()
+            || !part
+                .chars()
+                .all(|c: char| c.is_ascii_alphanumeric() || c == '_')
+        {
+            return None;
+        }
+        out.push('-');
+        out.push_str(part);
+    }
+    Some(out)
+}
+
+#[derive(Serialize)]
+struct Request<'a> {
+    q: &'a [String],
+    #[serde(skip_serializing_if = "Option::is_none")]
+    source: Option<&'a str>,
+    target: &'a str,
+    format: &'static str,
+}
+
+#[derive(Deserialize)]
+struct Response {
+    data: ResponseData,
+}
+
+#[derive(Deserialize)]
+struct ResponseData {
+    translations: Vec<Translation>,
+}
+
+#[derive(Deserialize)]
+struct Translation {
+    #[serde(rename = "translatedText")]
+    translated_text: String,
+}
+
+#[derive(Deserialize)]
+struct ApiError {
+    error: ApiErrorBody,
+}
+
+#[derive(Deserialize)]
+struct ApiErrorBody {
+    code: i64,
+    message: String,
+}
+
+impl Backend for Google {
+    fn id(&self) -> BackendId {
+        self.id.clone()
+    }
+
+    fn supports(&self, target: &Language) -> bool {
+        google_code(target).is_some()
+    }
+
+    fn characters_billed(&self) -> u64 {
+        self.billed.load(Ordering::Relaxed)
+    }
+
+    fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
+        Box::pin(async move {
+            let texts: Vec<String> = batch.cues.iter().map(|cue| cue.text.clone()).collect();
+
+            // Unreachable when called through `translate`, which checks
+            // `supports` first; kept honest so a direct caller still refuses.
+            let Some(target) = google_code(&batch.target) else {
+                return Err(Error::UnsupportedTarget {
+                    backend: self.id(),
+                    language: batch.target.clone(),
+                });
+            };
+            let source = google_source_code(&batch.source);
+
+            let mut url = self.base_url.clone();
+            url.query_pairs_mut()
+                .append_pair("key", &self.config.api_key);
+
+            let reply = self
+                .http
+                .post(url)
+                .json(&Request {
+                    q: &texts,
+                    source: source.as_deref(),
+                    target: &target,
+                    format: "text",
+                })
+                .send()
+                .await
+                .map_err(|error| Error::Transport {
+                    backend: self.id(),
+                    source: Box::new(error),
+                })?;
+
+            let status = reply.status();
+            match status {
+                StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }),
+                StatusCode::TOO_MANY_REQUESTS => {
+                    return Err(Error::RateLimited {
+                        backend: self.id(),
+                        retry_after: None,
+                    });
+                }
+                _ => {}
+            }
+
+            let bytes = reply.bytes().await.map_err(|error| Error::Transport {
+                backend: self.id(),
+                source: Box::new(error),
+            })?;
+            if !status.is_success() {
+                let detail = api_error_detail(status, &bytes);
+                if status.is_server_error() {
+                    return Err(Error::Transport {
+                        backend: self.id(),
+                        source: detail.into(),
+                    });
+                }
+                return Err(Error::Malformed {
+                    backend: self.id(),
+                    detail,
+                });
+            }
+
+            let parsed: Response =
+                serde_json::from_slice(&bytes).map_err(|error| Error::Malformed {
+                    backend: self.id(),
+                    detail: error.to_string(),
+                })?;
+
+            if parsed.data.translations.len() != texts.len() {
+                return Err(Error::CueMismatch {
+                    backend: self.id(),
+                    detail: format!(
+                        "sent {} texts, got {} translations back",
+                        texts.len(),
+                        parsed.data.translations.len()
+                    ),
+                });
+            }
+
+            self.billed.fetch_add(
+                texts.iter().map(|text| text.chars().count() as u64).sum(),
+                Ordering::Relaxed,
+            );
+
+            Ok(parsed
+                .data
+                .translations
+                .into_iter()
+                .zip(&batch.cues)
+                .map(|(translation, cue)| TranslatedCue {
+                    number: cue.number,
+                    // The API HTML-escapes even plain-text replies.
+                    text: unescape_html(&translation.translated_text),
+                })
+                .collect())
+        })
+    }
+}
+
+fn api_error_detail(status: StatusCode, body: &[u8]) -> String {
+    if let Ok(parsed) = serde_json::from_slice::<ApiError>(body) {
+        return format!("HTTP {}: {}", parsed.error.code, parsed.error.message);
+    }
+    let lossy = String::from_utf8_lossy(body);
+    let mut end = MAX_ERROR_BODY.min(lossy.len());
+    while !lossy.is_char_boundary(end) {
+        end -= 1;
+    }
+    format!("HTTP {status}: {}", &lossy[..end])
+}
+
+/// Decode the HTML entities Google's v2 replies carry even in text mode.
+///
+/// Named entities appear for quotes and ampersands (`&quot;`, `&#39;`,
+/// `&amp;`) and numeric ones for anything non-ASCII the encoder feels like
+/// escaping. Leaving them in would write visible `&#39;` into every subtitle
+/// line containing an apostrophe.
+#[must_use]
+pub fn unescape_html(text: &str) -> String {
+    if !text.contains('&') {
+        return text.to_owned();
+    }
+
+    let mut out = String::with_capacity(text.len());
+    let mut rest = text;
+
+    while let Some(position) = rest.find('&') {
+        out.push_str(&rest[..position]);
+        let tail = &rest[position..];
+        let semicolon = match tail.find(';') {
+            Some(index) if index <= 8 => index,
+            _ => {
+                out.push('&');
+                rest = &tail[1..];
+                continue;
+            }
+        };
+        let entity = &tail[1..semicolon];
+        if let Some(character) = named_entity(entity).or_else(|| numeric_entity(entity)) {
+            out.push(character);
+            rest = &tail[semicolon + 1..];
+        } else {
+            out.push('&');
+            rest = &tail[1..];
+        }
+    }
+    out.push_str(rest);
+    out
+}
+
+fn named_entity(entity: &str) -> Option<char> {
+    match entity {
+        "amp" => Some('&'),
+        "lt" => Some('<'),
+        "gt" => Some('>'),
+        "quot" => Some('"'),
+        "apos" => Some('\''),
+        _ => None,
+    }
+}
+
+fn numeric_entity(entity: &str) -> Option<char> {
+    let code = if let Some(hex) = entity.strip_prefix("#x").or(entity.strip_prefix("#X")) {
+        u32::from_str_radix(hex, 16).ok()?
+    } else {
+        entity.strip_prefix('#')?.parse::<u32>().ok()?
+    };
+    char::from_u32(code)
+}
+
+#[cfg(test)]
+mod tests {
+    use arr_core::Language;
+    use wiremock::matchers::method;
+    use wiremock::{Mock, MockServer, ResponseTemplate};
+
+    use super::{Google, GoogleConfig};
+    use crate::translate::{Backend, Batch, BatchCue, Error};
+
+    const KEY: &str = "test-api-key";
+
+    fn google(base_url: String) -> Google {
+        Google::with_base_url(
+            GoogleConfig {
+                api_key: KEY.to_owned(),
+            },
+            base_url,
+        )
+        .expect("client builds")
+    }
+
+    fn batch(source: Language, target: Language, texts: &[&str]) -> Batch {
+        Batch {
+            source,
+            target,
+            cues: texts
+                .iter()
+                .enumerate()
+                .map(|(index, text)| BatchCue {
+                    number: index + 1,
+                    text: (*text).to_owned(),
+                })
+                .collect(),
+        }
+    }
+
+    fn reply_body(translations: &[&str]) -> String {
+        let items: Vec<String> = translations
+            .iter()
+            .map(|text| format!(r#"{{"translatedText":{text}}}"#))
+            .collect();
+        format!(
+            r#"{{"data":{{"translations":[{items}]}}}}"#,
+            items = items.join(",")
+        )
+    }
+
+    async fn mount_translate(server: &MockServer, status: u16, body: &str) {
+        // The base URL *is* the endpoint under test, so only the method
+        // discriminates here.
+        Mock::given(method("POST"))
+            .respond_with(ResponseTemplate::new(status).set_body_string(body))
+            .mount(server)
+            .await;
+    }
+
+    #[test]
+    fn pt_is_brazilian_so_pt_pt_refuses_rather_than_pretends() {
+        let client = google("http://localhost".to_owned());
+        assert!(client.supports(&Language::PortugueseBrazil));
+        assert!(client.supports(&Language::Other("pt".to_owned())));
+        assert!(client.supports(&Language::Other("en".to_owned())));
+        assert!(!client.supports(&Language::PortuguesePortugal));
+        assert!(!client.supports(&Language::PortugueseUnverified));
+    }
+
+    #[tokio::test]
+    async fn an_unsupported_target_never_reaches_the_wire() {
+        let server = MockServer::start().await;
+        let client = google(server.uri());
+
+        let error = client
+            .translate(&batch(
+                Language::Other("en".to_owned()),
+                Language::PortuguesePortugal,
+                &["hi"],
+            ))
+            .await
+            .expect_err("pt-PT must be refused outright");
+
+        assert!(matches!(error, Error::UnsupportedTarget { .. }));
+        assert!(server
+            .received_requests()
+            .await
+            .expect("requests")
+            .is_empty());
+    }
+
+    #[tokio::test]
+    async fn a_batch_translates_with_numbers_preserved_and_entities_decoded() {
+        let server = MockServer::start().await;
+        mount_translate(
+            &server,
+            200,
+            &reply_body(&[r#""Ol\u00e1 &quot;mundo&quot;&#33;""#]),
+        )
+        .await;
+        let client = google(server.uri());
+
+        let reply = client
+            .translate(&batch(
+                Language::Other("en".to_owned()),
+                Language::PortugueseBrazil,
+                &["Hello world!"],
+            ))
+            .await
+            .expect("the mock answers");
+
+        assert_eq!(reply.len(), 1);
+        assert_eq!(reply[0].number, 1);
+        assert_eq!(reply[0].text, "Olá \"mundo\"!");
+
+        let request = &server.received_requests().await.expect("requests recorded")[0];
+        assert_eq!(
+            request
+                .url
+                .query_pairs()
+                .find(|(key, _)| key == "key")
+                .map(|(_, value)| value),
+            Some(std::borrow::Cow::Borrowed(KEY))
+        );
+        let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body");
+        assert_eq!(body["target"], "pt");
+        assert_eq!(body["source"], "en");
+        assert_eq!(body["format"], "text");
+    }
+
+    #[tokio::test]
+    async fn a_portuguese_source_hints_pt_without_claiming_a_variant() {
+        let server = MockServer::start().await;
+        mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await;
+        let client = google(server.uri());
+
+        client
+            .translate(&batch(
+                Language::PortuguesePortugal,
+                Language::Other("en".to_owned()),
+                &["Olá"],
+            ))
+            .await
+            .expect("pt-PT is a fine source");
+
+        let request = &server.received_requests().await.expect("requests recorded")[0];
+        let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body");
+        assert_eq!(body["source"], "pt");
+    }
+
+    #[tokio::test]
+    async fn characters_sent_are_reported_as_billed() {
+        let server = MockServer::start().await;
+        mount_translate(&server, 200, &reply_body(&[r#""x""#])).await;
+        let client = google(server.uri());
+        assert_eq!(client.characters_billed(), 0);
+
+        client
+            .translate(&batch(
+                Language::Other("en".to_owned()),
+                Language::PortugueseBrazil,
+                &["four"],
+            ))
+            .await
+            .expect("the mock answers");
+
+        assert_eq!(client.characters_billed(), 4);
+    }
+
+    #[tokio::test]
+    async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() {
+        let server = MockServer::start().await;
+        mount_translate(&server, 200, &reply_body(&[r#""um""#])).await;
+        let client = google(server.uri());
+
+        let error = client
+            .translate(&batch(
+                Language::Other("en".to_owned()),
+                Language::PortugueseBrazil,
+                &["one", "two"],
+            ))
+            .await
+            .expect_err("two texts in, one out must fail");
+
+        assert!(matches!(error, Error::CueMismatch { .. }));
+    }
+
+    #[tokio::test]
+    async fn a_bad_key_is_unauthorized_and_not_worth_retrying() {
+        let server = MockServer::start().await;
+        mount_translate(
+            &server,
+            403,
+            r#"{"error":{"code":403,"message":"API key not valid"}}"#,
+        )
+        .await;
+        let client = google(server.uri());
+
+        let error = client
+            .translate(&batch(
+                Language::Other("en".to_owned()),
+                Language::PortugueseBrazil,
+                &["hi"],
+            ))
+            .await
+            .expect_err("403 must be refused");
+
+        assert!(matches!(error, Error::Unauthorized { .. }));
+        assert!(!error.is_transient());
+    }
+
+    #[tokio::test]
+    async fn a_rate_limit_surfaces_as_the_queue_state() {
+        let server = MockServer::start().await;
+        mount_translate(&server, 429, "").await;
+        let client = google(server.uri());
+
+        let error = client
+            .translate(&batch(
+                Language::Other("en".to_owned()),
+                Language::PortugueseBrazil,
+                &["hi"],
+            ))
+            .await
+            .expect_err("429 must surface as RateLimited");
+
+        assert!(matches!(error, Error::RateLimited { .. }));
+        assert!(error.is_transient());
+    }
+
+    #[test]
+    fn html_entities_decode_including_numeric_ones() {
+        use super::unescape_html;
+        assert_eq!(unescape_html("plain"), "plain");
+        assert_eq!(unescape_html("a &amp; b"), "a & b");
+        assert_eq!(unescape_html("&quot;x&quot;"), "\"x\"");
+        assert_eq!(unescape_html("n&#39;then"), "n'then");
+        assert_eq!(unescape_html("caf&#233;"), "café");
+        assert_eq!(unescape_html("caf&#xE9;"), "café");
+        // A stray ampersand that is not an entity stays one.
+        assert_eq!(unescape_html("fish & chips"), "fish & chips");
+    }
+}
@@ -13,7 +13,7 @@
 //! Translation backends each sit behind their own cargo feature —
 //! `translate-openai`, `translate-deepl`, `translate-google`,
 //! `translate-command`. The features are declared; the `openai` and
-//! `command` backends are later issues (#191, #193), while DeepL
+//! `command` backends are later issues (#191, #193), while `DeepL`
 //! (`deepl`) and Google Translate (`google`) live here behind
 //! their flags. What they share — the [`translate::Backend`]
 //! trait, cue chunking, reply validation — lives in [`translate`], and the SRT