577 lines
18 KiB
Rust
577 lines
18 KiB
Rust
//! Translation: the layer between a subtitle and whichever backend renders it
|
||
//! into the wanted language.
|
||
//!
|
||
//! DESIGN.md §15. The backends — OpenAI-compatible, `DeepL`, Google Translate,
|
||
//! a remote command — are separate issues (#191, #192, #193), each behind its
|
||
//! own cargo feature. What lives here is everything they share, so that no
|
||
//! backend can get it wrong on its own:
|
||
//!
|
||
//! * **Chunking.** The unit sent out is a batch of cues, not a whole file and
|
||
//! not one cue. A movie is 40–80k characters and a season several hundred
|
||
//! thousand; one request per file is fragile, one per cue loses the context
|
||
//! that makes dialogue translate well.
|
||
//! * **Validation.** A reply whose cue count or numbering does not match the
|
||
//! batch is rejected. A backend that silently merges or drops lines must
|
||
//! fail loudly, not produce a subtitle whose lines have drifted out of sync
|
||
//! with the picture.
|
||
//! * **Reassembly.** Timing data never leaves arr. Only text goes out, and
|
||
//! the translated text comes back onto the original timings.
|
||
//!
|
||
//! When to translate is the reconcile loop's decision (#196); which backend
|
||
//! runs is a database setting (#198). This module only knows how.
|
||
|
||
use std::{fmt, future::Future, pin::Pin, time::Duration};
|
||
|
||
use arr_core::Language;
|
||
|
||
use crate::srt::Cue;
|
||
|
||
/// How many characters of cue text one batch carries at most.
|
||
///
|
||
/// The balance chunking strikes: batches small enough that one failure
|
||
/// re-does little and a length-limited backend is never overflowed, large
|
||
/// enough that a scene's dialogue travels together and a movie is tens of
|
||
/// requests, not thousands. Backends with tighter limits override
|
||
/// [`Backend::batch_budget`].
|
||
pub const BATCH_CHAR_BUDGET: usize = 4_096;
|
||
|
||
/// The name a translation backend answers to in settings and the UI.
|
||
///
|
||
/// A string rather than an enum for the same reason as
|
||
/// [`ProviderId`](crate::ProviderId): backends arrive one issue at a time,
|
||
/// each behind its own cargo feature, and which are compiled in is a
|
||
/// deployment fact.
|
||
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||
pub struct BackendId(String);
|
||
|
||
impl BackendId {
|
||
/// Name a backend.
|
||
#[must_use]
|
||
pub fn new(name: impl Into<String>) -> Self {
|
||
Self(name.into())
|
||
}
|
||
|
||
/// The name as written.
|
||
#[must_use]
|
||
pub fn as_str(&self) -> &str {
|
||
&self.0
|
||
}
|
||
}
|
||
|
||
impl fmt::Display for BackendId {
|
||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||
formatter.write_str(&self.0)
|
||
}
|
||
}
|
||
|
||
/// One numbered line of text on its way out. Number and text, nothing else:
|
||
/// timings stay home (DESIGN.md §15).
|
||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||
pub struct BatchCue {
|
||
/// The cue's 1-based position in the whole subtitle, continuous across
|
||
/// batches. The reply must quote it back unchanged.
|
||
pub number: usize,
|
||
/// The text to translate.
|
||
pub text: String,
|
||
}
|
||
|
||
/// One batch of cues put to one backend.
|
||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||
pub struct Batch {
|
||
/// The language the cues are in.
|
||
///
|
||
/// Explicit rather than auto-detected: the source is a subtitle arr
|
||
/// already holds, so its language is a recorded fact, and a backend
|
||
/// guessing it wrong is a silent way to mistranslate.
|
||
pub source: Language,
|
||
/// The language wanted back. pt-PT and pt-BR are distinct targets; a
|
||
/// backend that cannot express the difference refuses in
|
||
/// [`Backend::supports`] rather than quietly returning the wrong one.
|
||
pub target: Language,
|
||
/// The cues, in subtitle order.
|
||
pub cues: Vec<BatchCue>,
|
||
}
|
||
|
||
/// One translated cue in a backend's reply.
|
||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||
pub struct TranslatedCue {
|
||
/// The number of the [`BatchCue`] this translates.
|
||
pub number: usize,
|
||
/// The translated text.
|
||
pub text: String,
|
||
}
|
||
|
||
/// The translated batch, in reply order.
|
||
pub type TranslateFuture<'a> =
|
||
Pin<Box<dyn Future<Output = Result<Vec<TranslatedCue>>> + Send + 'a>>;
|
||
|
||
/// One translation backend (DESIGN.md §15).
|
||
///
|
||
/// Boxed futures rather than `async fn` for the same reason as
|
||
/// [`Provider`](crate::Provider): which backend runs is a database setting,
|
||
/// so callers hold a `dyn Backend` chosen at runtime.
|
||
///
|
||
/// A backend translates one batch and reports what happened. It does not
|
||
/// chunk, does not validate its own replies — [`translate`] does both, so a
|
||
/// backend cannot skip them — and never sees a timestamp.
|
||
pub trait Backend: fmt::Debug + Send + Sync {
|
||
/// The name this backend answers to in settings and the UI.
|
||
fn id(&self) -> BackendId;
|
||
|
||
/// Whether this backend can translate into `target` — exactly, variants
|
||
/// respected. A backend that only knows generic Portuguese answers
|
||
/// `false` for pt-PT rather than returning pt-BR under its name.
|
||
fn supports(&self, target: &Language) -> bool;
|
||
|
||
/// The most characters of cue text one batch to this backend may carry.
|
||
fn batch_budget(&self) -> usize {
|
||
BATCH_CHAR_BUDGET
|
||
}
|
||
|
||
/// Characters this backend has billed so far, cumulative over its life.
|
||
///
|
||
/// Translators charge per character (DESIGN.md §15); the daily budget
|
||
/// (#197) spends against what was actually billed rather than an
|
||
/// estimate of it. A backend that cannot say reports zero.
|
||
fn characters_billed(&self) -> u64 {
|
||
0
|
||
}
|
||
|
||
/// Translate one batch.
|
||
///
|
||
/// The reply is expected to carry every cue number the batch carried,
|
||
/// in order; [`translate`] rejects anything else. A backend that cannot
|
||
/// tell which of its output lines answers which input — a truncated or
|
||
/// refused reply, say — returns an error rather than guessing.
|
||
///
|
||
/// # Errors
|
||
///
|
||
/// Anything in [`Error`].
|
||
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a>;
|
||
}
|
||
|
||
/// Result alias for translation.
|
||
pub type Result<T> = std::result::Result<T, Error>;
|
||
|
||
/// Everything that can go wrong translating a subtitle.
|
||
///
|
||
/// Kept apart from the provider [`Error`](crate::Error): the two report to
|
||
/// different settings rows and different budget buckets, and a variant like
|
||
/// [`Error::CueMismatch`] has no provider counterpart.
|
||
#[derive(Debug, thiserror::Error)]
|
||
#[non_exhaustive]
|
||
pub enum Error {
|
||
/// The backend could not be reached at all, or the connection died.
|
||
#[error("{backend} is unreachable")]
|
||
Transport {
|
||
/// The backend that was called.
|
||
backend: BackendId,
|
||
/// The underlying transport failure.
|
||
#[source]
|
||
source: Box<dyn std::error::Error + Send + Sync>,
|
||
},
|
||
|
||
/// The backend refused the credentials it was given.
|
||
#[error("{backend} rejected the configured credentials")]
|
||
Unauthorized {
|
||
/// The backend that was called.
|
||
backend: BackendId,
|
||
},
|
||
|
||
/// The backend's rate limit or the daily budget was hit. A queue state,
|
||
/// not a failure (DESIGN.md §15).
|
||
#[error("{backend} is at its cap")]
|
||
RateLimited {
|
||
/// The backend that was called.
|
||
backend: BackendId,
|
||
/// How long the backend asked to be left alone, when it says.
|
||
retry_after: Option<Duration>,
|
||
},
|
||
|
||
/// The backend cannot translate into the requested language variant.
|
||
///
|
||
/// Raised by [`translate`] before anything is sent, off
|
||
/// [`Backend::supports`]. Configuration, not weather: it never resolves
|
||
/// by retrying.
|
||
#[error("{backend} cannot translate into {language}")]
|
||
UnsupportedTarget {
|
||
/// The backend that was asked.
|
||
backend: BackendId,
|
||
/// The language it cannot express.
|
||
language: Language,
|
||
},
|
||
|
||
/// The backend answered, but not in the shape a reply must have.
|
||
#[error("{backend} answered with something unexpected: {detail}")]
|
||
Malformed {
|
||
/// The backend that was called.
|
||
backend: BackendId,
|
||
/// What was wrong, short enough to log.
|
||
detail: String,
|
||
},
|
||
|
||
/// The reply's cue count or numbering does not match the batch.
|
||
///
|
||
/// The rejection DESIGN.md §15 demands: a backend that merged, dropped or
|
||
/// invented lines fails the whole translation loudly, and no partially
|
||
/// translated subtitle exists to be written.
|
||
#[error("{backend} returned a mismatched batch: {detail}")]
|
||
CueMismatch {
|
||
/// The backend that was called.
|
||
backend: BackendId,
|
||
/// Which cue diverged, and how.
|
||
detail: String,
|
||
},
|
||
}
|
||
|
||
impl Error {
|
||
/// Whether calling the same backend again later is worth doing.
|
||
///
|
||
/// A cap lifts and a network recovers. Bad credentials, an unsupported
|
||
/// target and a backend that mangles batches do not fix themselves, and
|
||
/// retrying them only burns budget.
|
||
#[must_use]
|
||
pub const fn is_transient(&self) -> bool {
|
||
matches!(self, Self::Transport { .. } | Self::RateLimited { .. })
|
||
}
|
||
}
|
||
|
||
/// Translate a whole subtitle through one backend.
|
||
///
|
||
/// Chunks the cues into batches within [`Backend::batch_budget`], sends them
|
||
/// in order, validates every reply against its batch, and reassembles the
|
||
/// translated text onto the original timings. All or nothing: any failed
|
||
/// batch fails the whole call, so a partially translated subtitle never
|
||
/// exists (DESIGN.md §15).
|
||
///
|
||
/// # Errors
|
||
///
|
||
/// [`Error::UnsupportedTarget`] before anything is sent when the backend
|
||
/// cannot express `target`; [`Error::CueMismatch`] when a reply does not
|
||
/// match its batch; otherwise whatever the backend reported.
|
||
pub async fn translate(
|
||
backend: &dyn Backend,
|
||
cues: &[Cue],
|
||
source: &Language,
|
||
target: &Language,
|
||
) -> Result<Vec<Cue>> {
|
||
if !backend.supports(target) {
|
||
return Err(Error::UnsupportedTarget {
|
||
backend: backend.id(),
|
||
language: target.clone(),
|
||
});
|
||
}
|
||
|
||
let mut texts = Vec::with_capacity(cues.len());
|
||
for batch in batches(cues, source, target, backend.batch_budget()) {
|
||
let reply = backend.translate(&batch).await?;
|
||
validate(&batch, &reply, &backend.id())?;
|
||
texts.extend(reply.into_iter().map(|cue| cue.text));
|
||
}
|
||
|
||
Ok(cues
|
||
.iter()
|
||
.zip(texts)
|
||
.map(|(cue, text)| Cue {
|
||
start: cue.start,
|
||
end: cue.end,
|
||
text,
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// Chunk cues into batches of at most `budget` characters of text.
|
||
///
|
||
/// Order preserved, numbering 1-based and continuous across batches. A single
|
||
/// cue longer than the budget still travels, alone in its own batch — a
|
||
/// too-long line is the backend's problem to refuse, not a reason to drop it.
|
||
fn batches(cues: &[Cue], source: &Language, target: &Language, budget: usize) -> Vec<Batch> {
|
||
let mut out: Vec<Batch> = Vec::new();
|
||
let mut current: Vec<BatchCue> = Vec::new();
|
||
let mut spent = 0;
|
||
|
||
for (position, cue) in cues.iter().enumerate() {
|
||
let cost = cue.text.chars().count();
|
||
if !current.is_empty() && spent + cost > budget {
|
||
out.push(Batch {
|
||
source: source.clone(),
|
||
target: target.clone(),
|
||
cues: std::mem::take(&mut current),
|
||
});
|
||
spent = 0;
|
||
}
|
||
current.push(BatchCue {
|
||
number: position + 1,
|
||
text: cue.text.clone(),
|
||
});
|
||
spent += cost;
|
||
}
|
||
|
||
if !current.is_empty() {
|
||
out.push(Batch {
|
||
source: source.clone(),
|
||
target: target.clone(),
|
||
cues: current,
|
||
});
|
||
}
|
||
|
||
out
|
||
}
|
||
|
||
/// Reject a reply whose count or numbering does not match its batch.
|
||
fn validate(batch: &Batch, reply: &[TranslatedCue], backend: &BackendId) -> Result<()> {
|
||
let mismatch = |detail: String| {
|
||
Err(Error::CueMismatch {
|
||
backend: backend.clone(),
|
||
detail,
|
||
})
|
||
};
|
||
|
||
if reply.len() != batch.cues.len() {
|
||
return mismatch(format!(
|
||
"sent {} cues, got {} back",
|
||
batch.cues.len(),
|
||
reply.len()
|
||
));
|
||
}
|
||
|
||
for (sent, got) in batch.cues.iter().zip(reply) {
|
||
if sent.number != got.number {
|
||
return mismatch(format!(
|
||
"expected cue {}, reply says {}",
|
||
sent.number, got.number
|
||
));
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use std::{
|
||
sync::atomic::{AtomicUsize, Ordering},
|
||
time::Duration,
|
||
};
|
||
|
||
use arr_core::Language;
|
||
|
||
use super::{translate, Backend, BackendId, Batch, Error, TranslateFuture, TranslatedCue};
|
||
use crate::srt::Cue;
|
||
|
||
/// Uppercases every cue and records how it was called; misbehaves on cue.
|
||
#[derive(Debug, Default)]
|
||
struct StubBackend {
|
||
budget: Option<usize>,
|
||
calls: AtomicUsize,
|
||
drop_last_cue: bool,
|
||
renumber_from: Option<usize>,
|
||
}
|
||
|
||
impl Backend for StubBackend {
|
||
fn id(&self) -> BackendId {
|
||
BackendId::new("stub")
|
||
}
|
||
|
||
fn supports(&self, target: &Language) -> bool {
|
||
*target != Language::PortugueseBrazil
|
||
}
|
||
|
||
fn batch_budget(&self) -> usize {
|
||
self.budget.unwrap_or(super::BATCH_CHAR_BUDGET)
|
||
}
|
||
|
||
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
|
||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||
Box::pin(async move {
|
||
let mut reply: Vec<TranslatedCue> = batch
|
||
.cues
|
||
.iter()
|
||
.map(|cue| TranslatedCue {
|
||
number: self.renumber_from.unwrap_or(cue.number),
|
||
text: cue.text.to_uppercase(),
|
||
})
|
||
.collect();
|
||
if self.drop_last_cue {
|
||
reply.pop();
|
||
}
|
||
Ok(reply)
|
||
})
|
||
}
|
||
}
|
||
|
||
fn cues(texts: &[&str]) -> Vec<Cue> {
|
||
texts
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(index, text)| Cue {
|
||
start: Duration::from_secs(index as u64),
|
||
end: Duration::from_secs(index as u64 + 1),
|
||
text: (*text).to_owned(),
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn text_is_translated_and_timings_never_move() {
|
||
let backend = StubBackend::default();
|
||
let source = cues(&["one", "two"]);
|
||
|
||
let translated = translate(
|
||
&backend,
|
||
&source,
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
)
|
||
.await
|
||
.expect("the stub translates");
|
||
|
||
assert_eq!(translated.len(), 2);
|
||
assert_eq!(translated[0].text, "ONE");
|
||
assert_eq!(translated[1].text, "TWO");
|
||
assert_eq!(translated[0].start, source[0].start);
|
||
assert_eq!(translated[1].end, source[1].end);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn the_budget_splits_cues_into_more_than_one_batch() {
|
||
let backend = StubBackend {
|
||
budget: Some(6),
|
||
..StubBackend::default()
|
||
};
|
||
// 4 + 4 chars overflow a budget of 6, so two batches.
|
||
let source = cues(&["aaaa", "bbbb", "cc"]);
|
||
|
||
let translated = translate(
|
||
&backend,
|
||
&source,
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
)
|
||
.await
|
||
.expect("both batches answer");
|
||
|
||
assert_eq!(backend.calls.load(Ordering::SeqCst), 2);
|
||
assert_eq!(translated[2].text, "CC");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn an_unsupported_target_is_refused_before_anything_is_sent() {
|
||
let backend = StubBackend::default();
|
||
|
||
let error = translate(
|
||
&backend,
|
||
&cues(&["hi"]),
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortugueseBrazil,
|
||
)
|
||
.await
|
||
.expect_err("the stub cannot say pt-BR");
|
||
|
||
assert!(matches!(error, Error::UnsupportedTarget { .. }));
|
||
assert_eq!(backend.calls.load(Ordering::SeqCst), 0);
|
||
assert!(!error.is_transient());
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn a_dropped_cue_fails_the_whole_translation() {
|
||
let backend = StubBackend {
|
||
drop_last_cue: true,
|
||
..StubBackend::default()
|
||
};
|
||
|
||
let error = translate(
|
||
&backend,
|
||
&cues(&["one", "two"]),
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
)
|
||
.await
|
||
.expect_err("a short reply must be rejected");
|
||
|
||
assert!(matches!(error, Error::CueMismatch { .. }));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn renumbered_cues_fail_the_whole_translation() {
|
||
let backend = StubBackend {
|
||
renumber_from: Some(99),
|
||
..StubBackend::default()
|
||
};
|
||
|
||
let error = translate(
|
||
&backend,
|
||
&cues(&["one"]),
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
)
|
||
.await
|
||
.expect_err("wrong numbering must be rejected");
|
||
|
||
assert!(matches!(error, Error::CueMismatch { .. }));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn an_empty_subtitle_translates_to_nothing_without_a_call() {
|
||
let backend = StubBackend::default();
|
||
|
||
let translated = translate(
|
||
&backend,
|
||
&[],
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
)
|
||
.await
|
||
.expect("nothing to send");
|
||
|
||
assert!(translated.is_empty());
|
||
assert_eq!(backend.calls.load(Ordering::SeqCst), 0);
|
||
}
|
||
|
||
#[test]
|
||
fn numbering_is_continuous_across_batches() {
|
||
let source = cues(&["aaaa", "bbbb", "cc"]);
|
||
let batches = super::batches(
|
||
&source,
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
6,
|
||
);
|
||
|
||
assert_eq!(batches.len(), 2);
|
||
assert_eq!(batches[0].cues[0].number, 1);
|
||
assert_eq!(batches[1].cues[0].number, 2);
|
||
assert_eq!(batches[1].cues[1].number, 3);
|
||
}
|
||
|
||
#[test]
|
||
fn a_cue_over_the_budget_travels_alone_rather_than_being_dropped() {
|
||
let source = cues(&["this line is far over budget", "b"]);
|
||
let batches = super::batches(
|
||
&source,
|
||
&Language::Other("en".to_owned()),
|
||
&Language::PortuguesePortugal,
|
||
4,
|
||
);
|
||
|
||
assert_eq!(batches.len(), 2);
|
||
assert_eq!(batches[0].cues.len(), 1);
|
||
assert_eq!(batches[1].cues.len(), 1);
|
||
}
|
||
|
||
#[test]
|
||
fn a_cap_is_worth_retrying_and_a_mangled_batch_is_not() {
|
||
let backend = BackendId::new("stub");
|
||
assert!(Error::RateLimited {
|
||
backend: backend.clone(),
|
||
retry_after: None,
|
||
}
|
||
.is_transient());
|
||
assert!(!Error::CueMismatch {
|
||
backend,
|
||
detail: "sent 2 cues, got 1 back".to_owned(),
|
||
}
|
||
.is_transient());
|
||
}
|
||
}
|