//! The Google Translate translation backend (`DESIGN.md` §15, issue #192). //! //! The cheap, ubiquitous option — and the reason #190's contract insists a //! backend own its language support: Google Translate v2's `pt` target is //! Brazilian in practice and it has no pt-PT at all. This module says so in //! [`Backend::supports`] rather than handing back Brazilian text under a //! pt-PT request. //! //! One request translates a whole batch: the texts go out as a `q` array and //! come back as a same-length `translations` array, order preserved, so cue //! numbering never leaves [`crate::translate`]. Two quirks of the API are //! handled here so callers never see them: //! //! * replies HTML-escape characters even with `format: "text"` //! (`"`, `'`), so they are decoded before returning; //! * per-character usage is not echoed back. #197's budget does not need it //! from here either — it counts source characters itself before this ever //! runs. //! //! The API key arrives from bootstrap config or environment per §10 — //! `ARR_TRANSLATE_GOOGLE_API_KEY`, plumbed through `arr-daemon`'s config — //! never a database row. use reqwest::{Client, StatusCode, Url}; use serde::{Deserialize, Serialize}; use arr_core::Language; use crate::translate::{Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue}; use crate::{Error as SubsError, ProviderId, Result}; /// The public Google Cloud Translation v2 endpoint. pub const DEFAULT_BASE_URL: &str = "https://translation.googleapis.com/language/translate/v2"; /// How much of an unexpected response body is worth keeping in an error. const MAX_ERROR_BODY: usize = 512; /// Credentials for Google Translate (DESIGN.md §10). #[derive(Clone, PartialEq, Eq)] pub struct GoogleConfig { /// The Cloud Translation API key, sent as the `key` query parameter. pub api_key: String, } // Hand-written: a derived Debug would print the key. impl std::fmt::Debug for GoogleConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("GoogleConfig") .field("api_key", &"") .finish() } } /// The Google Translate client behind the [`Backend`] trait. pub struct Google { http: Client, base_url: Url, config: GoogleConfig, id: BackendId, } // Hand-written: the key must never reach a log line. impl std::fmt::Debug for Google { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Google") .field("base_url", &self.base_url.as_str()) .finish_non_exhaustive() } } impl Google { /// A client against the real Google Translate API. /// /// # Errors /// /// When the HTTP client cannot be built or the default base URL does not /// parse. pub fn new(config: GoogleConfig) -> Result { Self::with_base_url(config, DEFAULT_BASE_URL) } /// A client against an arbitrary v2-compatible endpoint. Tests point this /// at a local mock; an operator with a proxy can do the same. /// /// # Errors /// /// When the HTTP client cannot be built or `base_url` does not parse. pub fn with_base_url(config: GoogleConfig, base_url: impl AsRef) -> Result { let url = Url::parse(base_url.as_ref()).map_err(|error| SubsError::Config { provider: ProviderId::new("google"), detail: error.to_string(), })?; Ok(Self { http: Client::new(), base_url: url, config, id: BackendId::new("google"), }) } } /// The code Google's `target` field expects, or `None` when Google cannot /// express the language honestly. /// /// `pt` is deliberately passed through bare: that is Google's one Portuguese /// target, and it is Brazilian. pt-PT and por-unverified refuse here. #[must_use] fn google_code(language: &Language) -> Option { match language { // Google's `pt` is Brazilian in practice; saying "pt-PT" to it would // still return Brazilian text under our name. Language::PortuguesePortugal | Language::PortugueseUnverified => None, Language::PortugueseBrazil => Some("pt".to_owned()), Language::Other(tag) => other_tag(tag), } } /// The `source` hint. Both Portuguese variants hint plain `pt`: a source /// describes text arr already holds, it promises nothing about output — and /// what output variant Google produces is exactly what [`google_code`] /// refuses to fake. #[must_use] fn google_source_code(language: &Language) -> Option { match language { Language::PortuguesePortugal | Language::PortugueseBrazil => Some("pt".to_owned()), Language::PortugueseUnverified => None, Language::Other(tag) => other_tag(tag), } } #[must_use] fn other_tag(tag: &str) -> Option { let tag = tag.trim(); let mut parts = tag.split('-'); let primary = parts.next()?; if !(2..=3).contains(&primary.len()) || !primary.chars().all(|c: char| c.is_ascii_alphabetic()) { return None; } let mut out = primary.to_ascii_lowercase(); for part in parts { if part.is_empty() || !part .chars() .all(|c: char| c.is_ascii_alphanumeric() || c == '_') { return None; } out.push('-'); out.push_str(part); } Some(out) } #[derive(Serialize)] struct Request<'a> { q: &'a [String], #[serde(skip_serializing_if = "Option::is_none")] source: Option<&'a str>, target: &'a str, format: &'static str, } #[derive(Deserialize)] struct Response { data: ResponseData, } #[derive(Deserialize)] struct ResponseData { translations: Vec, } #[derive(Deserialize)] struct Translation { #[serde(rename = "translatedText")] translated_text: String, } #[derive(Deserialize)] struct ApiError { error: ApiErrorBody, } #[derive(Deserialize)] struct ApiErrorBody { code: i64, message: String, } impl Backend for Google { fn id(&self) -> BackendId { self.id.clone() } fn supports(&self, target: &Language) -> bool { google_code(target).is_some() } fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> { Box::pin(async move { let texts: Vec = batch.cues.iter().map(|cue| cue.text.clone()).collect(); // Unreachable when called through `translate`, which checks // `supports` first; kept honest so a direct caller still refuses. let Some(target) = google_code(&batch.target) else { return Err(Error::UnsupportedTarget { backend: self.id(), language: batch.target.clone(), }); }; let source = google_source_code(&batch.source); let mut url = self.base_url.clone(); url.query_pairs_mut() .append_pair("key", &self.config.api_key); let reply = self .http .post(url) .json(&Request { q: &texts, source: source.as_deref(), target: &target, format: "text", }) .send() .await .map_err(|error| Error::Transport { backend: self.id(), source: Box::new(error), })?; let status = reply.status(); match status { StatusCode::FORBIDDEN => return Err(Error::Unauthorized { backend: self.id() }), StatusCode::TOO_MANY_REQUESTS => { return Err(Error::RateLimited { backend: self.id(), retry_after: None, }); } _ => {} } let bytes = reply.bytes().await.map_err(|error| Error::Transport { backend: self.id(), source: Box::new(error), })?; if !status.is_success() { let detail = api_error_detail(status, &bytes); if status.is_server_error() { return Err(Error::Transport { backend: self.id(), source: detail.into(), }); } return Err(Error::Malformed { backend: self.id(), detail, }); } let parsed: Response = serde_json::from_slice(&bytes).map_err(|error| Error::Malformed { backend: self.id(), detail: error.to_string(), })?; if parsed.data.translations.len() != texts.len() { return Err(Error::CueMismatch { backend: self.id(), detail: format!( "sent {} texts, got {} translations back", texts.len(), parsed.data.translations.len() ), }); } Ok(parsed .data .translations .into_iter() .zip(&batch.cues) .map(|(translation, cue)| TranslatedCue { number: cue.number, // The API HTML-escapes even plain-text replies. text: unescape_html(&translation.translated_text), }) .collect()) }) } } fn api_error_detail(status: StatusCode, body: &[u8]) -> String { if let Ok(parsed) = serde_json::from_slice::(body) { return format!("HTTP {}: {}", parsed.error.code, parsed.error.message); } let lossy = String::from_utf8_lossy(body); let mut end = MAX_ERROR_BODY.min(lossy.len()); while !lossy.is_char_boundary(end) { end -= 1; } format!("HTTP {status}: {}", &lossy[..end]) } /// Decode the HTML entities Google's v2 replies carry even in text mode. /// /// Named entities appear for quotes and ampersands (`"`, `'`, /// `&`) and numeric ones for anything non-ASCII the encoder feels like /// escaping. Leaving them in would write visible `'` into every subtitle /// line containing an apostrophe. #[must_use] pub fn unescape_html(text: &str) -> String { if !text.contains('&') { return text.to_owned(); } let mut out = String::with_capacity(text.len()); let mut rest = text; while let Some(position) = rest.find('&') { out.push_str(&rest[..position]); let tail = &rest[position..]; let semicolon = match tail.find(';') { Some(index) if index <= 8 => index, _ => { out.push('&'); rest = &tail[1..]; continue; } }; let entity = &tail[1..semicolon]; if let Some(character) = named_entity(entity).or_else(|| numeric_entity(entity)) { out.push(character); rest = &tail[semicolon + 1..]; } else { out.push('&'); rest = &tail[1..]; } } out.push_str(rest); out } fn named_entity(entity: &str) -> Option { match entity { "amp" => Some('&'), "lt" => Some('<'), "gt" => Some('>'), "quot" => Some('"'), "apos" => Some('\''), _ => None, } } fn numeric_entity(entity: &str) -> Option { let code = if let Some(hex) = entity.strip_prefix("#x").or(entity.strip_prefix("#X")) { u32::from_str_radix(hex, 16).ok()? } else { entity.strip_prefix('#')?.parse::().ok()? }; char::from_u32(code) } #[cfg(test)] mod tests { use arr_core::Language; use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; use super::{Google, GoogleConfig}; use crate::translate::{Backend, Batch, BatchCue, Error}; const KEY: &str = "test-api-key"; fn google(base_url: String) -> Google { Google::with_base_url( GoogleConfig { api_key: KEY.to_owned(), }, base_url, ) .expect("client builds") } fn batch(source: Language, target: Language, texts: &[&str]) -> Batch { Batch { source, target, cues: texts .iter() .enumerate() .map(|(index, text)| BatchCue { number: index + 1, text: (*text).to_owned(), }) .collect(), } } fn reply_body(translations: &[&str]) -> String { let items: Vec = translations .iter() .map(|text| format!(r#"{{"translatedText":{text}}}"#)) .collect(); format!( r#"{{"data":{{"translations":[{items}]}}}}"#, items = items.join(",") ) } async fn mount_translate(server: &MockServer, status: u16, body: &str) { // The base URL *is* the endpoint under test, so only the method // discriminates here. Mock::given(method("POST")) .respond_with(ResponseTemplate::new(status).set_body_string(body)) .mount(server) .await; } #[test] fn pt_is_brazilian_so_pt_pt_refuses_rather_than_pretends() { let client = google("http://localhost".to_owned()); assert!(client.supports(&Language::PortugueseBrazil)); assert!(client.supports(&Language::Other("pt".to_owned()))); assert!(client.supports(&Language::Other("en".to_owned()))); assert!(!client.supports(&Language::PortuguesePortugal)); assert!(!client.supports(&Language::PortugueseUnverified)); } #[tokio::test] async fn an_unsupported_target_never_reaches_the_wire() { let server = MockServer::start().await; let client = google(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortuguesePortugal, &["hi"], )) .await .expect_err("pt-PT must be refused outright"); assert!(matches!(error, Error::UnsupportedTarget { .. })); assert!(server .received_requests() .await .expect("requests") .is_empty()); } #[tokio::test] async fn a_batch_translates_with_numbers_preserved_and_entities_decoded() { let server = MockServer::start().await; mount_translate( &server, 200, &reply_body(&[r#""Ol\u00e1 "mundo"!""#]), ) .await; let client = google(server.uri()); let reply = client .translate(&batch( Language::Other("en".to_owned()), Language::PortugueseBrazil, &["Hello world!"], )) .await .expect("the mock answers"); assert_eq!(reply.len(), 1); assert_eq!(reply[0].number, 1); assert_eq!(reply[0].text, "Olá \"mundo\"!"); let request = &server.received_requests().await.expect("requests recorded")[0]; assert_eq!( request .url .query_pairs() .find(|(key, _)| key == "key") .map(|(_, value)| value), Some(std::borrow::Cow::Borrowed(KEY)) ); let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); assert_eq!(body["target"], "pt"); assert_eq!(body["source"], "en"); assert_eq!(body["format"], "text"); } #[tokio::test] async fn a_portuguese_source_hints_pt_without_claiming_a_variant() { let server = MockServer::start().await; mount_translate(&server, 200, &reply_body(&[r#""Hi""#])).await; let client = google(server.uri()); client .translate(&batch( Language::PortuguesePortugal, Language::Other("en".to_owned()), &["Olá"], )) .await .expect("pt-PT is a fine source"); let request = &server.received_requests().await.expect("requests recorded")[0]; let body: serde_json::Value = serde_json::from_slice(&request.body).expect("json body"); assert_eq!(body["source"], "pt"); } #[tokio::test] async fn a_wrong_translation_count_is_a_mismatch_not_a_subtitle() { let server = MockServer::start().await; mount_translate(&server, 200, &reply_body(&[r#""um""#])).await; let client = google(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortugueseBrazil, &["one", "two"], )) .await .expect_err("two texts in, one out must fail"); assert!(matches!(error, Error::CueMismatch { .. })); } #[tokio::test] async fn a_bad_key_is_unauthorized_and_not_worth_retrying() { let server = MockServer::start().await; mount_translate( &server, 403, r#"{"error":{"code":403,"message":"API key not valid"}}"#, ) .await; let client = google(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortugueseBrazil, &["hi"], )) .await .expect_err("403 must be refused"); assert!(matches!(error, Error::Unauthorized { .. })); assert!(!error.is_transient()); } #[tokio::test] async fn a_rate_limit_surfaces_as_the_queue_state() { let server = MockServer::start().await; mount_translate(&server, 429, "").await; let client = google(server.uri()); let error = client .translate(&batch( Language::Other("en".to_owned()), Language::PortugueseBrazil, &["hi"], )) .await .expect_err("429 must surface as RateLimited"); assert!(matches!(error, Error::RateLimited { .. })); assert!(error.is_transient()); } #[test] fn html_entities_decode_including_numeric_ones() { use super::unescape_html; assert_eq!(unescape_html("plain"), "plain"); assert_eq!(unescape_html("a & b"), "a & b"); assert_eq!(unescape_html(""x""), "\"x\""); assert_eq!(unescape_html("n'then"), "n'then"); assert_eq!(unescape_html("café"), "café"); assert_eq!(unescape_html("café"), "café"); // A stray ampersand that is not an entity stays one. assert_eq!(unescape_html("fish & chips"), "fish & chips"); } }