feat: run manual translations in the background (#275)
This commit was merged in pull request #275.
This commit is contained in:
@@ -24,6 +24,7 @@ mod state;
|
||||
mod subtitle_settings;
|
||||
mod subtitles;
|
||||
mod trailer;
|
||||
mod translations;
|
||||
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
@@ -60,6 +61,7 @@ pub use subtitles::{
|
||||
SubtitleSearchResults, SubtitleStatus, SubtitleTranslateInput,
|
||||
};
|
||||
pub use trailer::{Trailer, TrailerKind};
|
||||
pub use translations::{Translation, TranslationState};
|
||||
|
||||
/// Where the generated document is served, and where `just gen-client` reads
|
||||
/// it back from when it is fetched rather than dumped from the binary.
|
||||
@@ -414,6 +416,7 @@ mod tests {
|
||||
"SubtitleStatus",
|
||||
"MissingSubtitle",
|
||||
"EpisodeSubtitleStatus",
|
||||
"Translation",
|
||||
] {
|
||||
assert!(
|
||||
json["components"]["schemas"][schema].is_object(),
|
||||
|
||||
@@ -13,6 +13,7 @@ use tokio::sync::RwLock;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
use crate::jellyfin::JellyfinClient;
|
||||
use crate::translations::Translations;
|
||||
|
||||
/// The TMDB API root. Not a bootstrap setting (DESIGN.md §10) — only the key
|
||||
/// is configurable, so this is a constant that tests point elsewhere.
|
||||
@@ -111,6 +112,8 @@ pub struct AppState {
|
||||
syncer: Syncer,
|
||||
qbit: Option<QbitClient>,
|
||||
download_snapshot: DownloadSnapshot,
|
||||
/// Manual translations running as their own tasks (§15).
|
||||
translations: Translations,
|
||||
}
|
||||
|
||||
/// What the daemon reports back for a manual grab: `Err` carries the reason
|
||||
@@ -203,6 +206,7 @@ impl AppState {
|
||||
syncer: Syncer::default(),
|
||||
qbit: None,
|
||||
download_snapshot: Arc::new(RwLock::new(None)),
|
||||
translations: Translations::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -323,6 +327,10 @@ impl AppState {
|
||||
&self.ffmpeg_binary
|
||||
}
|
||||
|
||||
pub(crate) fn translations(&self) -> &Translations {
|
||||
&self.translations
|
||||
}
|
||||
|
||||
pub(crate) fn syncer(&self) -> &Syncer {
|
||||
&self.syncer
|
||||
}
|
||||
|
||||
+340
-125
@@ -42,6 +42,7 @@ use utoipa::ToSchema;
|
||||
use crate::movies::{pool, ApiError, ErrorBody};
|
||||
use crate::policies::parsed;
|
||||
use crate::state::AppState;
|
||||
use crate::translations::Translation;
|
||||
|
||||
/// One subtitle arr knows about, as the API renders it.
|
||||
#[derive(Debug, Clone, Serialize, ToSchema)]
|
||||
@@ -193,6 +194,9 @@ pub struct SubtitleStatus {
|
||||
/// sidecar (§15, #260). Empty for a file probed before #189, whose
|
||||
/// `sub_tracks` carry a language and no codec.
|
||||
pub embedded_tracks: Vec<EmbeddedTrack>,
|
||||
/// Manual translations still running, or failed since the request that
|
||||
/// started them returned.
|
||||
pub translations: Vec<Translation>,
|
||||
}
|
||||
|
||||
/// One episode's media file, subtitles and gaps — the series-wide bulk
|
||||
@@ -206,6 +210,7 @@ pub struct EpisodeSubtitleStatus {
|
||||
pub subtitles: Vec<Subtitle>,
|
||||
pub missing: Vec<MissingSubtitle>,
|
||||
pub embedded_tracks: Vec<EmbeddedTrack>,
|
||||
pub translations: Vec<Translation>,
|
||||
}
|
||||
|
||||
/// One wanted language a media file lacks, or a subtitle it has but `alass`
|
||||
@@ -765,6 +770,7 @@ async fn status_for_owner(
|
||||
subtitles,
|
||||
missing,
|
||||
embedded_tracks: embedded_tracks_of(state, id).await?,
|
||||
translations: state.translations().of(id),
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
@@ -868,6 +874,7 @@ pub async fn status_for_series(
|
||||
subtitles,
|
||||
missing,
|
||||
embedded_tracks: embedded_tracks_of(&state, row.media_file_id).await?,
|
||||
translations: state.translations().of(row.media_file_id),
|
||||
});
|
||||
}
|
||||
Ok(Json(out))
|
||||
@@ -1183,7 +1190,7 @@ pub async fn extract(
|
||||
params(("media_file_id" = i64, Path, description = "Media file row id")),
|
||||
request_body = SubtitleTranslateInput,
|
||||
responses(
|
||||
(status = 201, body = Subtitle),
|
||||
(status = 202, body = Translation),
|
||||
(status = 404, body = ErrorBody),
|
||||
(status = 409, body = ErrorBody),
|
||||
(status = 422, body = ErrorBody),
|
||||
@@ -1191,11 +1198,14 @@ pub async fn extract(
|
||||
(status = 503, body = ErrorBody)
|
||||
)
|
||||
)]
|
||||
/// Check the request, then translate as a task of its own and answer at
|
||||
/// once. The run reports on the file's subtitle status: `translations`
|
||||
/// while it runs or once it fails, the sidecar on `subtitles` once it lands.
|
||||
pub async fn translate(
|
||||
State(state): State<AppState>,
|
||||
UrlPath(media_file_id): UrlPath<i64>,
|
||||
body: Result<Json<SubtitleTranslateInput>, JsonRejection>,
|
||||
) -> Result<(StatusCode, Json<Subtitle>), ApiError> {
|
||||
) -> Result<(StatusCode, Json<Translation>), ApiError> {
|
||||
let input = parsed(body)?;
|
||||
let target_language = language_of(&input.target_language)?;
|
||||
let target = target(&state, media_file_id).await?;
|
||||
@@ -1227,31 +1237,103 @@ pub async fn translate(
|
||||
claim_language(&state, target.media_file_id, &target_language).await?;
|
||||
claim_path(&state, &destination).await?;
|
||||
|
||||
let raw = tokio::fs::read_to_string(&source_path)
|
||||
.await
|
||||
.map_err(|error| ApiError::Filesystem(format!("{source_path}: {error}")))?;
|
||||
let cues = arr_subs::srt::parse(&raw)
|
||||
.map_err(|error| ApiError::Invalid(format!("source_subtitle_id: not SRT: {error}")))?;
|
||||
let translated =
|
||||
arr_subs::translate::translate(backend.as_ref(), &cues, &source_language, &target_language)
|
||||
let tag = target_language.to_string();
|
||||
let run = state
|
||||
.translations()
|
||||
.start(media_file_id, &tag, &engine)
|
||||
.ok_or_else(|| ApiError::Conflict(format!("{tag} is already being translated")))?;
|
||||
|
||||
tokio::spawn(async move {
|
||||
let job = TranslationJob {
|
||||
target,
|
||||
source_path,
|
||||
source_language,
|
||||
target_language,
|
||||
destination,
|
||||
engine,
|
||||
backend,
|
||||
};
|
||||
match job.run(&state).await {
|
||||
Ok(()) => state.translations().finish(media_file_id, &tag),
|
||||
Err(detail) => {
|
||||
tracing::warn!(media_file_id, language = %tag, %detail, "manual translation failed");
|
||||
state.translations().fail(media_file_id, &tag, detail);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok((StatusCode::ACCEPTED, Json(run)))
|
||||
}
|
||||
|
||||
/// Everything a manual translation needs once the request has been checked.
|
||||
struct TranslationJob {
|
||||
target: Target,
|
||||
source_path: String,
|
||||
source_language: Language,
|
||||
target_language: Language,
|
||||
destination: PathBuf,
|
||||
engine: String,
|
||||
backend: Arc<dyn arr_subs::Backend>,
|
||||
}
|
||||
|
||||
impl TranslationJob {
|
||||
/// The reason it failed, in words for the panel.
|
||||
async fn run(self, state: &AppState) -> Result<(), String> {
|
||||
let raw = tokio::fs::read_to_string(&self.source_path)
|
||||
.await
|
||||
.map_err(|error| ApiError::SubtitleUpstream(error.to_string()))?;
|
||||
.map_err(|error| format!("{}: {error}", self.source_path))?;
|
||||
let cues = arr_subs::srt::parse(&raw)
|
||||
.map_err(|error| format!("source subtitle is not SRT: {error}"))?;
|
||||
let translated = arr_subs::translate::translate(
|
||||
self.backend.as_ref(),
|
||||
&cues,
|
||||
&self.source_language,
|
||||
&self.target_language,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
||||
write_sidecar(&destination, &arr_subs::srt::render(&translated)).await?;
|
||||
write_sidecar(&self.destination, &arr_subs::srt::render(&translated))
|
||||
.await
|
||||
.map_err(failure)?;
|
||||
|
||||
let sync = state.syncer().settle(&target.path, &destination).await;
|
||||
if let Some(synced) = &sync.content {
|
||||
write_sidecar(&destination, synced).await?;
|
||||
let sync = state
|
||||
.syncer()
|
||||
.settle(&self.target.path, &self.destination)
|
||||
.await;
|
||||
if let Some(synced) = &sync.content {
|
||||
write_sidecar(&self.destination, synced)
|
||||
.await
|
||||
.map_err(failure)?;
|
||||
}
|
||||
|
||||
let record = arr_db::NewSubtitleFile::translated(
|
||||
self.target.media_file_id,
|
||||
&self.target_language.to_string(),
|
||||
&self.engine,
|
||||
&self.destination.to_string_lossy(),
|
||||
)
|
||||
.sync(db_sync_state(sync.state));
|
||||
finish(
|
||||
state,
|
||||
&record,
|
||||
self.target.media_file_id,
|
||||
&self.target_language,
|
||||
)
|
||||
.await
|
||||
.map(drop)
|
||||
.map_err(failure)
|
||||
}
|
||||
}
|
||||
|
||||
let record = arr_db::NewSubtitleFile::translated(
|
||||
target.media_file_id,
|
||||
&target_language.to_string(),
|
||||
&engine,
|
||||
&destination.to_string_lossy(),
|
||||
)
|
||||
.sync(db_sync_state(sync.state));
|
||||
finish(&state, &record, target.media_file_id, &target_language).await
|
||||
/// An [`ApiError`] from the write half of a translation, which has no
|
||||
/// response left to carry it.
|
||||
fn failure(error: ApiError) -> String {
|
||||
match error {
|
||||
ApiError::Filesystem(detail) | ApiError::Conflict(detail) => detail,
|
||||
ApiError::Database(detail) => format!("database error: {detail}"),
|
||||
other => format!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
@@ -2196,6 +2278,57 @@ mod tests {
|
||||
(status, response.json().await.expect("grab json"))
|
||||
}
|
||||
|
||||
/// Start a translation and wait for it the way the panel does, by
|
||||
/// polling the title's subtitle status. An accepted run answers with the
|
||||
/// subtitle it wrote, or with the run itself once it failed.
|
||||
async fn translate(
|
||||
fixture: &Fixture,
|
||||
body: serde_json::Value,
|
||||
) -> (StatusCode, serde_json::Value) {
|
||||
let client = reqwest::Client::new();
|
||||
let target = body["target_language"].clone();
|
||||
let response = client
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
let status = response.status();
|
||||
let started: serde_json::Value = response.json().await.expect("translate json");
|
||||
if status != StatusCode::ACCEPTED {
|
||||
return (status, started);
|
||||
}
|
||||
for _ in 0..200 {
|
||||
let files: serde_json::Value = client
|
||||
.get(format!("{}/api/movies/1/subtitles/status", fixture.base))
|
||||
.send()
|
||||
.await
|
||||
.expect("status")
|
||||
.json()
|
||||
.await
|
||||
.expect("status json");
|
||||
let file = &files[0];
|
||||
let runs = file["translations"].as_array().expect("translations");
|
||||
if let Some(run) = runs.iter().find(|run| run["language"] == target) {
|
||||
if run["state"] == "failed" {
|
||||
return (status, run.clone());
|
||||
}
|
||||
} else if let Some(written) = file["subtitles"]
|
||||
.as_array()
|
||||
.expect("subtitles")
|
||||
.iter()
|
||||
.find(|subtitle| subtitle["language"] == target)
|
||||
{
|
||||
return (status, written.clone());
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("translation into {target} never settled");
|
||||
}
|
||||
|
||||
fn pt() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"provider": "opensubtitles",
|
||||
@@ -2421,20 +2554,16 @@ mod tests {
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
|
||||
// `.en.mt.srt` — English is satisfied by a machine translation.
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source["id"].as_i64().expect("source id"),
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED, "{body}");
|
||||
|
||||
// A provider fetch for English would write `.en.srt`, a name nothing
|
||||
// holds. §15 refuses it anyway, and says how to replace it instead.
|
||||
@@ -2633,21 +2762,16 @@ mod tests {
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
let source_id = source["id"].as_i64().expect("source id");
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source_id,
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
let body: serde_json::Value = response.json().await.expect("json");
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED, "{body}");
|
||||
assert_eq!(body["origin"], "translated");
|
||||
assert_eq!(body["engine"], "openai");
|
||||
|
||||
@@ -2686,21 +2810,16 @@ mod tests {
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
let source_id = source["id"].as_i64().expect("source id");
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source_id,
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
let body: serde_json::Value = response.json().await.expect("json");
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED, "{body}");
|
||||
assert_eq!(body["sync"], "synced");
|
||||
let sidecar = PathBuf::from(body["path"].as_str().expect("path"));
|
||||
assert_eq!(
|
||||
@@ -2718,31 +2837,26 @@ mod tests {
|
||||
)
|
||||
.await;
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
let client = reqwest::Client::new();
|
||||
let translate = |id: i64, target: &str| {
|
||||
client
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
"source_subtitle_id": id,
|
||||
"target_language": target,
|
||||
"engine": "openai"
|
||||
}))
|
||||
.send()
|
||||
};
|
||||
|
||||
let first: serde_json::Value = translate(source["id"].as_i64().expect("id"), "en")
|
||||
.await
|
||||
.expect("translate")
|
||||
.json()
|
||||
.await
|
||||
.expect("json");
|
||||
let second = translate(first["id"].as_i64().expect("id"), "es")
|
||||
.await
|
||||
.expect("translate again");
|
||||
assert_eq!(second.status(), StatusCode::CREATED);
|
||||
let (_, first) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source["id"].as_i64().expect("id"),
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let (status, second) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": first["id"].as_i64().expect("id"),
|
||||
"target_language": "es",
|
||||
"engine": "openai"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED);
|
||||
assert_eq!(second["origin"], "translated", "{second}");
|
||||
}
|
||||
|
||||
/// An embedded track has no file, so it carries no text to translate.
|
||||
@@ -2756,20 +2870,16 @@ mod tests {
|
||||
.await
|
||||
.expect("record embedded");
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": id,
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}");
|
||||
}
|
||||
|
||||
/// The engines §15 names are not all compiled in; one that is not must
|
||||
@@ -2779,20 +2889,16 @@ mod tests {
|
||||
let fixture = application(vec![Arc::new(StubProvider::new("opensubtitles"))], vec![]).await;
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source["id"],
|
||||
"target_language": "en",
|
||||
"engine": "deepl"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2800,19 +2906,133 @@ mod tests {
|
||||
let fixture = application(vec![Arc::new(StubProvider::new("opensubtitles"))], vec![]).await;
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source["id"],
|
||||
"target_language": "en"
|
||||
}))
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}");
|
||||
}
|
||||
|
||||
/// Fails every batch, the way a model reply with the wrong cue count does.
|
||||
#[derive(Debug)]
|
||||
struct FailingBackend;
|
||||
|
||||
impl Backend for FailingBackend {
|
||||
fn id(&self) -> BackendId {
|
||||
BackendId::new("openai")
|
||||
}
|
||||
|
||||
fn supports(&self, _target: &Language) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn translate<'a>(&'a self, _batch: &'a Batch) -> TranslateFuture<'a> {
|
||||
Box::pin(async move {
|
||||
Err(arr_subs::translate::Error::Malformed {
|
||||
backend: BackendId::new("openai"),
|
||||
detail: "status 400: model unloaded".into(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn probe(&self) -> arr_subs::translate::ProbeFuture<'_> {
|
||||
Box::pin(async move { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Never answers, so a run stays in flight for as long as a test needs.
|
||||
#[derive(Debug)]
|
||||
struct StuckBackend;
|
||||
|
||||
impl Backend for StuckBackend {
|
||||
fn id(&self) -> BackendId {
|
||||
BackendId::new("openai")
|
||||
}
|
||||
|
||||
fn supports(&self, _target: &Language) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn translate<'a>(&'a self, _batch: &'a Batch) -> TranslateFuture<'a> {
|
||||
Box::pin(std::future::pending())
|
||||
}
|
||||
|
||||
fn probe(&self) -> arr_subs::translate::ProbeFuture<'_> {
|
||||
Box::pin(async move { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// A failure lands after the request returned, so the status carries it
|
||||
/// for the panel, and nothing is written.
|
||||
#[tokio::test]
|
||||
async fn a_failed_translation_is_reported_on_the_status_and_writes_nothing() {
|
||||
let fixture = application(
|
||||
vec![Arc::new(StubProvider::new("opensubtitles"))],
|
||||
vec![Arc::new(FailingBackend)],
|
||||
)
|
||||
.await;
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
|
||||
let (status, run) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source["id"],
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED);
|
||||
assert_eq!(run["state"], "failed", "{run}");
|
||||
let detail = run["detail"].as_str().expect("detail");
|
||||
assert!(detail.contains("model unloaded"), "{detail}");
|
||||
assert_eq!(fixture.subtitle_rows().await.len(), 1, "only the source");
|
||||
assert!(!fixture
|
||||
.folder
|
||||
.join("Dune (2021) [tmdbid-438631] - [2160p][WEB-DL].en.mt.srt")
|
||||
.exists());
|
||||
}
|
||||
|
||||
/// The run outlives the request, and a second click on the same
|
||||
/// language is refused rather than starting a second writer.
|
||||
#[tokio::test]
|
||||
async fn a_running_translation_answers_at_once_and_refuses_a_second() {
|
||||
let fixture = application(
|
||||
vec![Arc::new(StubProvider::new("opensubtitles"))],
|
||||
vec![Arc::new(StuckBackend)],
|
||||
)
|
||||
.await;
|
||||
let (_, source) = grab(&fixture, pt()).await;
|
||||
let body = serde_json::json!({
|
||||
"source_subtitle_id": source["id"],
|
||||
"target_language": "en",
|
||||
"engine": "openai"
|
||||
});
|
||||
let client = reqwest::Client::new();
|
||||
let url = format!("{}/api/media-files/1/subtitles/translate", fixture.base);
|
||||
|
||||
let first = client.post(&url).json(&body).send().await.expect("first");
|
||||
assert_eq!(first.status(), StatusCode::ACCEPTED);
|
||||
let run: serde_json::Value = first.json().await.expect("json");
|
||||
assert_eq!(run["state"], "running");
|
||||
|
||||
let second = client.post(&url).json(&body).send().await.expect("second");
|
||||
assert_eq!(second.status(), StatusCode::CONFLICT);
|
||||
|
||||
let files: serde_json::Value = client
|
||||
.get(format!("{}/api/movies/1/subtitles/status", fixture.base))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
.expect("status")
|
||||
.json()
|
||||
.await
|
||||
.expect("status json");
|
||||
assert_eq!(files[0]["translations"][0]["language"], "en");
|
||||
assert_eq!(files[0]["translations"][0]["state"], "running");
|
||||
}
|
||||
|
||||
/// Deleting removes both halves, and the language becomes a gap again —
|
||||
@@ -3188,21 +3408,16 @@ mod tests {
|
||||
let (status, source) = extract(&fixture, 0).await;
|
||||
assert_eq!(status, StatusCode::CREATED, "{source}");
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/api/media-files/1/subtitles/translate",
|
||||
fixture.base
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
let (status, body) = translate(
|
||||
&fixture,
|
||||
serde_json::json!({
|
||||
"source_subtitle_id": source["id"].as_i64().expect("source id"),
|
||||
"target_language": "pt-PT",
|
||||
"engine": "openai"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("translate");
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
let body: serde_json::Value = response.json().await.expect("json");
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::ACCEPTED, "{body}");
|
||||
assert_eq!(body["origin"], "translated");
|
||||
assert_eq!(body["language"], "pt-PT");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
//! Manual translations still running, or that failed after the request that
|
||||
//! started them had already returned (§15).
|
||||
//!
|
||||
//! A translation through a local model can take minutes, longer than anyone
|
||||
//! keeps a page open, and a handler awaiting it is dropped the moment the
|
||||
//! browser goes away. So the translate endpoint answers at once and the work
|
||||
//! runs as its own task; this is where that task reports.
|
||||
//!
|
||||
//! Transient state in the sense of §8: kept in memory, never persisted, gone
|
||||
//! with the process. A restart mid-translation loses the run and writes
|
||||
//! nothing, which leaves the language exactly as it was before the click.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use serde::Serialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
/// Where one manual translation stands. A finished one is not here at all:
|
||||
/// its sidecar is on the file's subtitle list instead.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, ToSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TranslationState {
|
||||
Running,
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// One manual translation on one media file.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)]
|
||||
pub struct Translation {
|
||||
/// The language being written, as `arr_core::Language` spells it.
|
||||
pub language: String,
|
||||
pub engine: String,
|
||||
pub state: TranslationState,
|
||||
/// Why it failed. Set only when `state` is `failed`.
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
/// Every media file's manual translations. Cheap to clone; all clones share
|
||||
/// one map.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Translations {
|
||||
inner: Arc<Mutex<HashMap<i64, Vec<Translation>>>>,
|
||||
}
|
||||
|
||||
impl Translations {
|
||||
/// Record a run as started, unless one for the same language is already
|
||||
/// running on this file. A failed run for that language is replaced:
|
||||
/// starting again is how the operator retries it.
|
||||
pub fn start(&self, media_file_id: i64, language: &str, engine: &str) -> Option<Translation> {
|
||||
let mut map = self.lock();
|
||||
let runs = map.entry(media_file_id).or_default();
|
||||
if runs
|
||||
.iter()
|
||||
.any(|run| run.language == language && run.state == TranslationState::Running)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
runs.retain(|run| run.language != language);
|
||||
let run = Translation {
|
||||
language: language.to_owned(),
|
||||
engine: engine.to_owned(),
|
||||
state: TranslationState::Running,
|
||||
detail: None,
|
||||
};
|
||||
runs.push(run.clone());
|
||||
Some(run)
|
||||
}
|
||||
|
||||
/// The run landed: its sidecar now speaks for it.
|
||||
pub fn finish(&self, media_file_id: i64, language: &str) {
|
||||
let mut map = self.lock();
|
||||
if let Some(runs) = map.get_mut(&media_file_id) {
|
||||
runs.retain(|run| run.language != language);
|
||||
if runs.is_empty() {
|
||||
map.remove(&media_file_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The run failed; keep the reason until the next attempt replaces it.
|
||||
pub fn fail(&self, media_file_id: i64, language: &str, detail: String) {
|
||||
let mut map = self.lock();
|
||||
if let Some(run) = map
|
||||
.get_mut(&media_file_id)
|
||||
.and_then(|runs| runs.iter_mut().find(|run| run.language == language))
|
||||
{
|
||||
run.state = TranslationState::Failed;
|
||||
run.detail = Some(detail);
|
||||
}
|
||||
}
|
||||
|
||||
/// This file's runs, ordered by language like the rest of its status.
|
||||
#[must_use]
|
||||
pub fn of(&self, media_file_id: i64) -> Vec<Translation> {
|
||||
let mut runs = self.lock().get(&media_file_id).cloned().unwrap_or_default();
|
||||
runs.sort_by(|a, b| a.language.cmp(&b.language));
|
||||
runs
|
||||
}
|
||||
|
||||
// Nothing holding the lock can leave the map half-written, so a panic
|
||||
// elsewhere is no reason to stop reporting.
|
||||
fn lock(&self) -> MutexGuard<'_, HashMap<i64, Vec<Translation>>> {
|
||||
self.inner.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{TranslationState, Translations};
|
||||
|
||||
#[test]
|
||||
fn a_second_run_for_a_running_language_is_refused() {
|
||||
let runs = Translations::default();
|
||||
assert!(runs.start(1, "en", "openai").is_some());
|
||||
assert!(runs.start(1, "en", "openai").is_none());
|
||||
assert!(
|
||||
runs.start(1, "es", "openai").is_some(),
|
||||
"other languages run"
|
||||
);
|
||||
assert!(runs.start(2, "en", "openai").is_some(), "other files run");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failure_stays_until_the_retry_replaces_it() {
|
||||
let runs = Translations::default();
|
||||
runs.start(1, "en", "openai");
|
||||
runs.fail(1, "en", "status 400".into());
|
||||
let failed = runs.of(1);
|
||||
assert_eq!(failed.len(), 1);
|
||||
assert_eq!(failed[0].state, TranslationState::Failed);
|
||||
assert_eq!(failed[0].detail.as_deref(), Some("status 400"));
|
||||
|
||||
assert!(runs.start(1, "en", "openai").is_some(), "a retry starts");
|
||||
let retried = runs.of(1);
|
||||
assert_eq!(retried.len(), 1);
|
||||
assert_eq!(retried[0].state, TranslationState::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_finished_run_leaves_nothing_behind() {
|
||||
let runs = Translations::default();
|
||||
runs.start(1, "en", "openai");
|
||||
runs.finish(1, "en");
|
||||
assert_eq!(runs.of(1), Vec::new());
|
||||
}
|
||||
}
|
||||
+79
-8
@@ -167,6 +167,7 @@ import {
|
||||
subtitleSummary,
|
||||
subtitleSummaryLabel,
|
||||
summaryWidth,
|
||||
type Translation,
|
||||
translateSubtitle,
|
||||
translationSources,
|
||||
wantedSubtitles,
|
||||
@@ -192,6 +193,8 @@ import {
|
||||
} from "./ui";
|
||||
|
||||
const POLL_MS = 15_000;
|
||||
/** How often an open title re-reads a running translation's status. */
|
||||
const TRANSLATION_POLL_MS = 5000;
|
||||
|
||||
/** §9.8: the live download snapshot, refetched on the app's one poll cadence
|
||||
* (POLL_MS) rather than a second timer. Read by row builders at paint time. */
|
||||
@@ -1059,7 +1062,11 @@ function downloadChip(
|
||||
const downloadSays = new WeakMap<HTMLElement, (text: string | null) => void>();
|
||||
|
||||
function fillDownloadSlot(slot: HTMLElement) {
|
||||
const download = findDownload(latestDownloads, slot.dataset.dlKind ?? "", Number(slot.dataset.dlId));
|
||||
const download = findDownload(
|
||||
latestDownloads,
|
||||
slot.dataset.dlKind ?? "",
|
||||
Number(slot.dataset.dlId),
|
||||
);
|
||||
if (download === null) {
|
||||
slot.replaceChildren();
|
||||
return;
|
||||
@@ -1503,6 +1510,9 @@ function subtitleSection(status: SubtitleStatus, refresh: () => void): SubtitleS
|
||||
let subtitles = status.subtitles;
|
||||
let missing = status.missing;
|
||||
let tracks = status.embedded_tracks;
|
||||
let runs = status.translations;
|
||||
/** Pending re-read while a translation runs; one per panel at most. */
|
||||
let pollTimer: number | undefined;
|
||||
/** Set by an extraction so the sidecar it just wrote stays picked across
|
||||
* the repaint that follows. */
|
||||
let pickAfterRepaint: number | null = null;
|
||||
@@ -1547,7 +1557,13 @@ function subtitleSection(status: SubtitleStatus, refresh: () => void): SubtitleS
|
||||
*/
|
||||
function paintChips() {
|
||||
const summary = subtitleSummary(
|
||||
{ media_file_id: mediaFileId, subtitles, missing, embedded_tracks: tracks },
|
||||
{
|
||||
media_file_id: mediaFileId,
|
||||
subtitles,
|
||||
missing,
|
||||
embedded_tracks: tracks,
|
||||
translations: runs,
|
||||
},
|
||||
wanted,
|
||||
);
|
||||
// the codes read as one string — the chip's own flex gap would otherwise
|
||||
@@ -1747,7 +1763,57 @@ function subtitleSection(status: SubtitleStatus, refresh: () => void): SubtitleS
|
||||
targetLanguages.bar(picked?.language ?? null);
|
||||
// a track is a source only once it has been written out, so until then
|
||||
// the button has nothing to translate from
|
||||
translateButton.disabled = engine.select.disabled || picked === null || "index" in picked;
|
||||
translateButton.disabled =
|
||||
engine.select.disabled || picked === null || "index" in picked || running().length > 0;
|
||||
}
|
||||
|
||||
function running(): Translation[] {
|
||||
return runs.filter((run) => run.state === "running");
|
||||
}
|
||||
|
||||
/**
|
||||
* Say where the file's translations stand, from the status rather than
|
||||
* from the click, so a run started before the page was left still shows.
|
||||
* `before` is the previous status's runs: a language that was running and
|
||||
* is now gone has landed.
|
||||
*/
|
||||
function paintRuns(before: Translation[]) {
|
||||
const now = running();
|
||||
const failed = runs.find((run) => run.state === "failed");
|
||||
const landed = before.find(
|
||||
(run) =>
|
||||
run.state === "running" &&
|
||||
!runs.some((next) => next.language === run.language) &&
|
||||
subtitles.some((subtitle) => subtitle.language === run.language),
|
||||
);
|
||||
if (now.length > 0) {
|
||||
translateNote.hidden = false;
|
||||
say(translateNote, `translating into ${now.map((run) => run.language).join(", ")}…`);
|
||||
} else if (failed !== undefined) {
|
||||
translateNote.hidden = false;
|
||||
say(
|
||||
translateNote,
|
||||
`translation into ${failed.language} failed — ${failed.detail ?? "no reason given"}`,
|
||||
"fault",
|
||||
);
|
||||
} else if (landed !== undefined) {
|
||||
translateNote.hidden = false;
|
||||
say(
|
||||
translateNote,
|
||||
`written next to the video as a machine translation into ${landed.language}`,
|
||||
);
|
||||
}
|
||||
window.clearTimeout(pollTimer);
|
||||
pollTimer = undefined;
|
||||
// a run takes minutes; re-read on a slow beat until it settles, and stop
|
||||
// once the panel has left the page
|
||||
if (now.length > 0) {
|
||||
pollTimer = window.setTimeout(() => {
|
||||
if (toggle.isConnected) {
|
||||
refresh();
|
||||
}
|
||||
}, TRANSLATION_POLL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1949,20 +2015,21 @@ function subtitleSection(status: SubtitleStatus, refresh: () => void): SubtitleS
|
||||
const sourceId = picked.id;
|
||||
translateButton.disabled = true;
|
||||
translateNote.hidden = false;
|
||||
say(translateNote, `translating into ${target}…`);
|
||||
say(translateNote, `starting a translation into ${target}…`);
|
||||
void translateSubtitle(mediaFileId, {
|
||||
source_subtitle_id: sourceId,
|
||||
target_language: target,
|
||||
engine: engine.select.value,
|
||||
}).then((outcome) => {
|
||||
translateButton.disabled = false;
|
||||
translateNote.hidden = false;
|
||||
if (outcome.kind === "error") {
|
||||
translateButton.disabled = false;
|
||||
say(translateNote, `translation failed — ${outcome.detail}`, "fault");
|
||||
return;
|
||||
}
|
||||
say(translateNote, `written next to the video as a machine translation into ${target}`);
|
||||
refresh();
|
||||
const before = runs;
|
||||
runs = [...runs.filter((run) => run.language !== outcome.run.language), outcome.run];
|
||||
paintSources();
|
||||
paintRuns(before);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1975,6 +2042,7 @@ function subtitleSection(status: SubtitleStatus, refresh: () => void): SubtitleS
|
||||
});
|
||||
|
||||
paintChips();
|
||||
paintRuns([]);
|
||||
// the wanted set decides which languages the chip may name at all, so it
|
||||
// is asked for on sight rather than when the panel opens; the request is
|
||||
// shared by every row on the page
|
||||
@@ -1989,8 +2057,11 @@ function subtitleSection(status: SubtitleStatus, refresh: () => void): SubtitleS
|
||||
subtitles = next.subtitles;
|
||||
missing = next.missing;
|
||||
tracks = next.embedded_tracks;
|
||||
const before = runs;
|
||||
runs = next.translations;
|
||||
paintChips();
|
||||
paintSources();
|
||||
paintRuns(before);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
+26
-3
@@ -53,6 +53,20 @@ export interface SubtitleStatus {
|
||||
missing: MissingSubtitle[];
|
||||
/** Empty for a file probed before #189, which recorded no codec. */
|
||||
embedded_tracks: EmbeddedTrack[];
|
||||
/** Manual translations still running, or failed after the click returned. */
|
||||
translations: Translation[];
|
||||
}
|
||||
|
||||
/**
|
||||
* A manual translation the daemon runs on its own (§15). A finished one is
|
||||
* not listed: its sidecar is on `subtitles` instead.
|
||||
*/
|
||||
export interface Translation {
|
||||
language: string;
|
||||
engine: string;
|
||||
state: "running" | "failed";
|
||||
/** Set only when `state` is `failed`. */
|
||||
detail: string | null;
|
||||
}
|
||||
|
||||
/** The series-wide bulk sibling of `SubtitleStatus` (one call, not one per episode). */
|
||||
@@ -258,11 +272,20 @@ export async function grabSubtitle(
|
||||
}
|
||||
}
|
||||
|
||||
/** Translate one subtitle already on the file into another language (§15). */
|
||||
export type TranslateOutcome =
|
||||
| { kind: "started"; run: Translation }
|
||||
| { kind: "error"; detail: string };
|
||||
|
||||
/**
|
||||
* Start translating one subtitle already on the file into another language
|
||||
* (§15). The daemon answers before the translation is done; the file's
|
||||
* subtitle status reports the run from then on, so leaving the page loses
|
||||
* nothing.
|
||||
*/
|
||||
export async function translateSubtitle(
|
||||
mediaFileId: number,
|
||||
input: { source_subtitle_id: number; target_language: string; engine: string },
|
||||
): Promise<SubtitleWriteOutcome> {
|
||||
): Promise<TranslateOutcome> {
|
||||
try {
|
||||
const response = await fetch(`/api/media-files/${mediaFileId}/subtitles/translate`, {
|
||||
method: "POST",
|
||||
@@ -272,7 +295,7 @@ export async function translateSubtitle(
|
||||
if (!response.ok) {
|
||||
return { kind: "error", detail: await errorDetail(response) };
|
||||
}
|
||||
return { kind: "done", subtitle: (await response.json()) as Subtitle };
|
||||
return { kind: "started", run: (await response.json()) as Translation };
|
||||
} catch {
|
||||
return { kind: "error", detail: "daemon unreachable" };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user