Files
arr/crates/arr-subs/src/translate.rs
T

577 lines
18 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Translation: the layer between a subtitle and whichever backend renders it
//! into the wanted language.
//!
//! DESIGN.md §15. The backends — OpenAI-compatible, `DeepL`, Google Translate,
//! a remote command — are separate issues (#191, #192, #193), each behind its
//! own cargo feature. What lives here is everything they share, so that no
//! backend can get it wrong on its own:
//!
//! * **Chunking.** The unit sent out is a batch of cues, not a whole file and
//! not one cue. A movie is 40–80k characters and a season several hundred
//! thousand; one request per file is fragile, one per cue loses the context
//! that makes dialogue translate well.
//! * **Validation.** A reply whose cue count or numbering does not match the
//! batch is rejected. A backend that silently merges or drops lines must
//! fail loudly, not produce a subtitle whose lines have drifted out of sync
//! with the picture.
//! * **Reassembly.** Timing data never leaves arr. Only text goes out, and
//! the translated text comes back onto the original timings.
//!
//! When to translate is the reconcile loop's decision (#196); which backend
//! runs is a database setting (#198). This module only knows how.
use std::{fmt, future::Future, pin::Pin, time::Duration};
use arr_core::Language;
use crate::srt::Cue;
/// How many characters of cue text one batch carries at most.
///
/// The balance chunking strikes: batches small enough that one failure
/// re-does little and a length-limited backend is never overflowed, large
/// enough that a scene's dialogue travels together and a movie is tens of
/// requests, not thousands. Backends with tighter limits override
/// [`Backend::batch_budget`].
pub const BATCH_CHAR_BUDGET: usize = 4_096;
/// The name a translation backend answers to in settings and the UI.
///
/// A string rather than an enum for the same reason as
/// [`ProviderId`](crate::ProviderId): backends arrive one issue at a time,
/// each behind its own cargo feature, and which are compiled in is a
/// deployment fact.
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct BackendId(String);
impl BackendId {
/// Name a backend.
#[must_use]
pub fn new(name: impl Into<String>) -> Self {
Self(name.into())
}
/// The name as written.
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for BackendId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
/// One numbered line of text on its way out. Number and text, nothing else:
/// timings stay home (DESIGN.md §15).
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BatchCue {
/// The cue's 1-based position in the whole subtitle, continuous across
/// batches. The reply must quote it back unchanged.
pub number: usize,
/// The text to translate.
pub text: String,
}
/// One batch of cues put to one backend.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Batch {
/// The language the cues are in.
///
/// Explicit rather than auto-detected: the source is a subtitle arr
/// already holds, so its language is a recorded fact, and a backend
/// guessing it wrong is a silent way to mistranslate.
pub source: Language,
/// The language wanted back. pt-PT and pt-BR are distinct targets; a
/// backend that cannot express the difference refuses in
/// [`Backend::supports`] rather than quietly returning the wrong one.
pub target: Language,
/// The cues, in subtitle order.
pub cues: Vec<BatchCue>,
}
/// One translated cue in a backend's reply.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TranslatedCue {
/// The number of the [`BatchCue`] this translates.
pub number: usize,
/// The translated text.
pub text: String,
}
/// The translated batch, in reply order.
pub type TranslateFuture<'a> =
Pin<Box<dyn Future<Output = Result<Vec<TranslatedCue>>> + Send + 'a>>;
/// One translation backend (DESIGN.md §15).
///
/// Boxed futures rather than `async fn` for the same reason as
/// [`Provider`](crate::Provider): which backend runs is a database setting,
/// so callers hold a `dyn Backend` chosen at runtime.
///
/// A backend translates one batch and reports what happened. It does not
/// chunk, does not validate its own replies — [`translate`] does both, so a
/// backend cannot skip them — and never sees a timestamp.
pub trait Backend: fmt::Debug + Send + Sync {
/// The name this backend answers to in settings and the UI.
fn id(&self) -> BackendId;
/// Whether this backend can translate into `target` — exactly, variants
/// respected. A backend that only knows generic Portuguese answers
/// `false` for pt-PT rather than returning pt-BR under its name.
fn supports(&self, target: &Language) -> bool;
/// The most characters of cue text one batch to this backend may carry.
fn batch_budget(&self) -> usize {
BATCH_CHAR_BUDGET
}
/// Characters this backend has billed so far, cumulative over its life.
///
/// Translators charge per character (DESIGN.md §15); the daily budget
/// (#197) spends against what was actually billed rather than an
/// estimate of it. A backend that cannot say reports zero.
fn characters_billed(&self) -> u64 {
0
}
/// Translate one batch.
///
/// The reply is expected to carry every cue number the batch carried,
/// in order; [`translate`] rejects anything else. A backend that cannot
/// tell which of its output lines answers which input — a truncated or
/// refused reply, say — returns an error rather than guessing.
///
/// # Errors
///
/// Anything in [`Error`].
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a>;
}
/// Result alias for translation.
pub type Result<T> = std::result::Result<T, Error>;
/// Everything that can go wrong translating a subtitle.
///
/// Kept apart from the provider [`Error`](crate::Error): the two report to
/// different settings rows and different budget buckets, and a variant like
/// [`Error::CueMismatch`] has no provider counterpart.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// The backend could not be reached at all, or the connection died.
#[error("{backend} is unreachable")]
Transport {
/// The backend that was called.
backend: BackendId,
/// The underlying transport failure.
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
/// The backend refused the credentials it was given.
#[error("{backend} rejected the configured credentials")]
Unauthorized {
/// The backend that was called.
backend: BackendId,
},
/// The backend's rate limit or the daily budget was hit. A queue state,
/// not a failure (DESIGN.md §15).
#[error("{backend} is at its cap")]
RateLimited {
/// The backend that was called.
backend: BackendId,
/// How long the backend asked to be left alone, when it says.
retry_after: Option<Duration>,
},
/// The backend cannot translate into the requested language variant.
///
/// Raised by [`translate`] before anything is sent, off
/// [`Backend::supports`]. Configuration, not weather: it never resolves
/// by retrying.
#[error("{backend} cannot translate into {language}")]
UnsupportedTarget {
/// The backend that was asked.
backend: BackendId,
/// The language it cannot express.
language: Language,
},
/// The backend answered, but not in the shape a reply must have.
#[error("{backend} answered with something unexpected: {detail}")]
Malformed {
/// The backend that was called.
backend: BackendId,
/// What was wrong, short enough to log.
detail: String,
},
/// The reply's cue count or numbering does not match the batch.
///
/// The rejection DESIGN.md §15 demands: a backend that merged, dropped or
/// invented lines fails the whole translation loudly, and no partially
/// translated subtitle exists to be written.
#[error("{backend} returned a mismatched batch: {detail}")]
CueMismatch {
/// The backend that was called.
backend: BackendId,
/// Which cue diverged, and how.
detail: String,
},
}
impl Error {
/// Whether calling the same backend again later is worth doing.
///
/// A cap lifts and a network recovers. Bad credentials, an unsupported
/// target and a backend that mangles batches do not fix themselves, and
/// retrying them only burns budget.
#[must_use]
pub const fn is_transient(&self) -> bool {
matches!(self, Self::Transport { .. } | Self::RateLimited { .. })
}
}
/// Translate a whole subtitle through one backend.
///
/// Chunks the cues into batches within [`Backend::batch_budget`], sends them
/// in order, validates every reply against its batch, and reassembles the
/// translated text onto the original timings. All or nothing: any failed
/// batch fails the whole call, so a partially translated subtitle never
/// exists (DESIGN.md §15).
///
/// # Errors
///
/// [`Error::UnsupportedTarget`] before anything is sent when the backend
/// cannot express `target`; [`Error::CueMismatch`] when a reply does not
/// match its batch; otherwise whatever the backend reported.
pub async fn translate(
backend: &dyn Backend,
cues: &[Cue],
source: &Language,
target: &Language,
) -> Result<Vec<Cue>> {
if !backend.supports(target) {
return Err(Error::UnsupportedTarget {
backend: backend.id(),
language: target.clone(),
});
}
let mut texts = Vec::with_capacity(cues.len());
for batch in batches(cues, source, target, backend.batch_budget()) {
let reply = backend.translate(&batch).await?;
validate(&batch, &reply, &backend.id())?;
texts.extend(reply.into_iter().map(|cue| cue.text));
}
Ok(cues
.iter()
.zip(texts)
.map(|(cue, text)| Cue {
start: cue.start,
end: cue.end,
text,
})
.collect())
}
/// Chunk cues into batches of at most `budget` characters of text.
///
/// Order preserved, numbering 1-based and continuous across batches. A single
/// cue longer than the budget still travels, alone in its own batch — a
/// too-long line is the backend's problem to refuse, not a reason to drop it.
fn batches(cues: &[Cue], source: &Language, target: &Language, budget: usize) -> Vec<Batch> {
let mut out: Vec<Batch> = Vec::new();
let mut current: Vec<BatchCue> = Vec::new();
let mut spent = 0;
for (position, cue) in cues.iter().enumerate() {
let cost = cue.text.chars().count();
if !current.is_empty() && spent + cost > budget {
out.push(Batch {
source: source.clone(),
target: target.clone(),
cues: std::mem::take(&mut current),
});
spent = 0;
}
current.push(BatchCue {
number: position + 1,
text: cue.text.clone(),
});
spent += cost;
}
if !current.is_empty() {
out.push(Batch {
source: source.clone(),
target: target.clone(),
cues: current,
});
}
out
}
/// Reject a reply whose count or numbering does not match its batch.
fn validate(batch: &Batch, reply: &[TranslatedCue], backend: &BackendId) -> Result<()> {
let mismatch = |detail: String| {
Err(Error::CueMismatch {
backend: backend.clone(),
detail,
})
};
if reply.len() != batch.cues.len() {
return mismatch(format!(
"sent {} cues, got {} back",
batch.cues.len(),
reply.len()
));
}
for (sent, got) in batch.cues.iter().zip(reply) {
if sent.number != got.number {
return mismatch(format!(
"expected cue {}, reply says {}",
sent.number, got.number
));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::{
sync::atomic::{AtomicUsize, Ordering},
time::Duration,
};
use arr_core::Language;
use super::{translate, Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue};
use crate::srt::Cue;
/// Uppercases every cue and records how it was called; misbehaves on cue.
#[derive(Debug, Default)]
struct StubBackend {
budget: Option<usize>,
calls: AtomicUsize,
drop_last_cue: bool,
renumber_from: Option<usize>,
}
impl Backend for StubBackend {
fn id(&self) -> BackendId {
BackendId::new("stub")
}
fn supports(&self, target: &Language) -> bool {
*target != Language::PortugueseBrazil
}
fn batch_budget(&self) -> usize {
self.budget.unwrap_or(super::BATCH_CHAR_BUDGET)
}
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
self.calls.fetch_add(1, Ordering::SeqCst);
Box::pin(async move {
let mut reply: Vec<TranslatedCue> = batch
.cues
.iter()
.map(|cue| TranslatedCue {
number: self.renumber_from.unwrap_or(cue.number),
text: cue.text.to_uppercase(),
})
.collect();
if self.drop_last_cue {
reply.pop();
}
Ok(reply)
})
}
}
fn cues(texts: &[&str]) -> Vec<Cue> {
texts
.iter()
.enumerate()
.map(|(index, text)| Cue {
start: Duration::from_secs(index as u64),
end: Duration::from_secs(index as u64 + 1),
text: (*text).to_owned(),
})
.collect()
}
#[tokio::test]
async fn text_is_translated_and_timings_never_move() {
let backend = StubBackend::default();
let source = cues(&["one", "two"]);
let translated = translate(
&backend,
&source,
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
)
.await
.expect("the stub translates");
assert_eq!(translated.len(), 2);
assert_eq!(translated[0].text, "ONE");
assert_eq!(translated[1].text, "TWO");
assert_eq!(translated[0].start, source[0].start);
assert_eq!(translated[1].end, source[1].end);
}
#[tokio::test]
async fn the_budget_splits_cues_into_more_than_one_batch() {
let backend = StubBackend {
budget: Some(6),
..StubBackend::default()
};
// 4 + 4 chars overflow a budget of 6, so two batches.
let source = cues(&["aaaa", "bbbb", "cc"]);
let translated = translate(
&backend,
&source,
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
)
.await
.expect("both batches answer");
assert_eq!(backend.calls.load(Ordering::SeqCst), 2);
assert_eq!(translated[2].text, "CC");
}
#[tokio::test]
async fn an_unsupported_target_is_refused_before_anything_is_sent() {
let backend = StubBackend::default();
let error = translate(
&backend,
&cues(&["hi"]),
&Language::Other("en".to_owned()),
&Language::PortugueseBrazil,
)
.await
.expect_err("the stub cannot say pt-BR");
assert!(matches!(error, Error::UnsupportedTarget { .. }));
assert_eq!(backend.calls.load(Ordering::SeqCst), 0);
assert!(!error.is_transient());
}
#[tokio::test]
async fn a_dropped_cue_fails_the_whole_translation() {
let backend = StubBackend {
drop_last_cue: true,
..StubBackend::default()
};
let error = translate(
&backend,
&cues(&["one", "two"]),
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
)
.await
.expect_err("a short reply must be rejected");
assert!(matches!(error, Error::CueMismatch { .. }));
}
#[tokio::test]
async fn renumbered_cues_fail_the_whole_translation() {
let backend = StubBackend {
renumber_from: Some(99),
..StubBackend::default()
};
let error = translate(
&backend,
&cues(&["one"]),
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
)
.await
.expect_err("wrong numbering must be rejected");
assert!(matches!(error, Error::CueMismatch { .. }));
}
#[tokio::test]
async fn an_empty_subtitle_translates_to_nothing_without_a_call() {
let backend = StubBackend::default();
let translated = translate(
&backend,
&[],
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
)
.await
.expect("nothing to send");
assert!(translated.is_empty());
assert_eq!(backend.calls.load(Ordering::SeqCst), 0);
}
#[test]
fn numbering_is_continuous_across_batches() {
let source = cues(&["aaaa", "bbbb", "cc"]);
let batches = super::batches(
&source,
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
6,
);
assert_eq!(batches.len(), 2);
assert_eq!(batches[0].cues[0].number, 1);
assert_eq!(batches[1].cues[0].number, 2);
assert_eq!(batches[1].cues[1].number, 3);
}
#[test]
fn a_cue_over_the_budget_travels_alone_rather_than_being_dropped() {
let source = cues(&["this line is far over budget", "b"]);
let batches = super::batches(
&source,
&Language::Other("en".to_owned()),
&Language::PortuguesePortugal,
4,
);
assert_eq!(batches.len(), 2);
assert_eq!(batches[0].cues.len(), 1);
assert_eq!(batches[1].cues.len(), 1);
}
#[test]
fn a_cap_is_worth_retrying_and_a_mangled_batch_is_not() {
let backend = BackendId::new("stub");
assert!(Error::RateLimited {
backend: backend.clone(),
retry_after: None,
}
.is_transient());
assert!(!Error::CueMismatch {
backend,
detail: "sent 2 cues, got 1 back".to_owned(),
}
.is_transient());
}
}