Miguel Palhas
6ee79519ca
Cross-process against the real arr binary, real ffmpeg and real alass: a provider fetch synced and named under DESIGN.md §15, never re-searched once satisfied; an embedded English track extracted and translated into a .mt sidecar; an implausible sync kept unsynced and surfaced in the missing-subtitles queue. Podnapisi and the translation backend are stubbed at the HTTP/process boundary, never a live tracker or a live translation API.
431 lines
16 KiB
Rust
431 lines
16 KiB
Rust
//! The subtitle path end to end (DESIGN.md §15, issue #205), cross-process
|
|
//! like the rest of this crate: the real `arr` binary, real `ffmpeg` and real
|
|
//! `alass`. Only the HTTP boundary — Podnapisi — and the remote-command
|
|
//! translation backend are stubbed, the same rule §12 already applies to
|
|
//! Prowlarr and TMDB: a live tracker or a live translation API never runs in
|
|
//! a test.
|
|
//!
|
|
//! There is no HTTP route that adopts an already-imported file (that
|
|
//! pipeline is its own seam — grab, download, import), so every scenario
|
|
//! here seeds a `media_files` row directly against the daemon's own SQLite
|
|
//! file, the same fixture shape the daemon's in-process subtitle tests use.
|
|
|
|
// Same per-target quirk as `e2e.rs`: dev-dependencies used only by this
|
|
// integration target still link into the crate's own test build.
|
|
use {arr_db as _, arr_dl as _, arr_indexer as _, arr_meta as _, chrono as _};
|
|
|
|
use std::os::unix::fs::PermissionsExt;
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::Duration;
|
|
|
|
use arr_e2e::{fixtures, probe_movie_fixture, Daemon};
|
|
use wiremock::matchers::{method, path};
|
|
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
|
|
/// Neither Prowlarr, TMDB nor Transmission matter to a subtitle scenario;
|
|
/// every test points them at an address nothing answers on rather than
|
|
/// standing up fakes it never calls.
|
|
const UNUSED_UPSTREAM: &str = "http://127.0.0.1:1";
|
|
|
|
/// Insert a movie, an imported grab and a media file directly into the
|
|
/// daemon's database, then copy `clip` into the daemon's media root next to
|
|
/// it. Returns the media file id and the video's path on disk.
|
|
async fn seed_movie_file(
|
|
database: &arr_db::Db,
|
|
media_root: &Path,
|
|
tmdb_id: i64,
|
|
release_name: &str,
|
|
probed: Option<&serde_json::Value>,
|
|
) -> (i64, PathBuf) {
|
|
let pool = database.pool();
|
|
|
|
let root_id: i64 =
|
|
sqlx::query_scalar("SELECT id FROM roots WHERE kind = 'movie' AND audience = 'main'")
|
|
.fetch_one(pool)
|
|
.await
|
|
.expect("migrations seed the main movie root");
|
|
|
|
let movie_id: i64 = sqlx::query_scalar(
|
|
"INSERT INTO movies (tmdb_id, title, root_id) VALUES (?, ?, ?) RETURNING id",
|
|
)
|
|
.bind(tmdb_id)
|
|
.bind(format!("Movie {tmdb_id}"))
|
|
.bind(root_id)
|
|
.fetch_one(pool)
|
|
.await
|
|
.expect("insert movie");
|
|
|
|
let folder = media_root.join(format!("Movie {tmdb_id} (2024) - [1080p]"));
|
|
std::fs::create_dir_all(&folder).expect("create title folder");
|
|
let video = folder.join(format!("Movie {tmdb_id} (2024) - [1080p].mkv"));
|
|
std::fs::copy(probe_movie_fixture(), &video).expect("copy the real probe clip");
|
|
let size = i64::try_from(std::fs::metadata(&video).expect("clip metadata").len())
|
|
.expect("clip size fits in i64");
|
|
|
|
let media_file_id: i64 = sqlx::query_scalar(
|
|
"INSERT INTO media_files (owner_kind, owner_id, path, size, probed)
|
|
VALUES ('movie', ?, ?, ?, ?) RETURNING id",
|
|
)
|
|
.bind(movie_id)
|
|
.bind(video.to_string_lossy().into_owned())
|
|
.bind(size)
|
|
.bind(probed.map(ToString::to_string))
|
|
.fetch_one(pool)
|
|
.await
|
|
.expect("insert media file");
|
|
|
|
// A release and an imported grab, so `Target::release_name` resolves to
|
|
// `release_name` — Podnapisi searches nothing without one.
|
|
let release_id: i64 = sqlx::query_scalar(
|
|
"INSERT INTO releases (indexer_id, guid, name, size, download_url, parsed, verdict)
|
|
VALUES (1, ?, ?, 1000000, 'http://example.invalid/download', '{}', 'eligible')
|
|
RETURNING id",
|
|
)
|
|
.bind(uuid::Uuid::new_v4().to_string())
|
|
.bind(release_name)
|
|
.fetch_one(pool)
|
|
.await
|
|
.expect("insert release");
|
|
|
|
sqlx::query(
|
|
"INSERT INTO grabs (release_id, target_kind, target_id, infohash, state, imported_at)
|
|
VALUES (?, 'movie', ?, ?, 'imported', strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))",
|
|
)
|
|
.bind(release_id)
|
|
.bind(movie_id)
|
|
.bind(uuid::Uuid::new_v4().to_string())
|
|
.execute(pool)
|
|
.await
|
|
.expect("insert grab");
|
|
|
|
(media_file_id, video)
|
|
}
|
|
|
|
/// `PUT /api/settings/subtitles`, the operator surface the reconcile loop
|
|
/// reads every tick (DESIGN.md §15, #198).
|
|
async fn configure_subtitles(
|
|
base: &str,
|
|
wanted_languages: &[&str],
|
|
providers_enabled: &[&str],
|
|
translation_engine: Option<&str>,
|
|
) {
|
|
let response = reqwest::Client::new()
|
|
.put(format!("{base}/api/settings/subtitles"))
|
|
.json(&serde_json::json!({
|
|
"wanted_languages": wanted_languages,
|
|
"providers_enabled": providers_enabled,
|
|
"translation_engine": translation_engine,
|
|
"provider_daily_budgets": {},
|
|
"translator_daily_budgets": {},
|
|
"remote_command_timeout_seconds": 30,
|
|
}))
|
|
.send()
|
|
.await
|
|
.expect("put subtitle settings");
|
|
assert_eq!(
|
|
response.status(),
|
|
reqwest::StatusCode::OK,
|
|
"subtitle settings accepted: {}",
|
|
response.text().await.unwrap_or_default()
|
|
);
|
|
}
|
|
|
|
/// Poll `/api/media-files/{id}/subtitles` until a row of the given `origin`
|
|
/// shows up — the reconcile lane closes a gap from a detached task, so its
|
|
/// effects land some time after the tick that started it (§8, §15).
|
|
async fn wait_for_subtitle(base: &str, media_file_id: i64, origin: &str) -> serde_json::Value {
|
|
for _ in 0..200 {
|
|
let subtitles: Vec<serde_json::Value> =
|
|
reqwest::get(format!("{base}/api/media-files/{media_file_id}/subtitles"))
|
|
.await
|
|
.expect("list subtitles")
|
|
.json()
|
|
.await
|
|
.expect("subtitles json");
|
|
if let Some(found) = subtitles
|
|
.iter()
|
|
.find(|subtitle| subtitle["origin"] == origin)
|
|
{
|
|
return found.clone();
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
}
|
|
panic!("no {origin} subtitle appeared for media file {media_file_id} in time");
|
|
}
|
|
|
|
/// A remote-command translation backend stub (DESIGN.md §15, issue #193):
|
|
/// discards whatever batch it is fed and always answers with the same
|
|
/// translated cue. Every scenario here feeds it exactly one embedded English
|
|
/// cue ("hello", numbered 1, from the committed probe fixture), so a fixed
|
|
/// reply exercises the real IPC without needing a JSON parser in `sh`.
|
|
fn write_stub_translator(dir: &Path) -> PathBuf {
|
|
let script = dir.join("stub-translate.sh");
|
|
std::fs::write(
|
|
&script,
|
|
"#!/bin/sh\ncat >/dev/null\nprintf '[{\"number\":1,\"text\":\"Ola\"}]'\n",
|
|
)
|
|
.expect("write stub translator");
|
|
let mut permissions = std::fs::metadata(&script)
|
|
.expect("stub translator metadata")
|
|
.permissions();
|
|
permissions.set_mode(0o755);
|
|
std::fs::set_permissions(&script, permissions).expect("make stub translator executable");
|
|
script
|
|
}
|
|
|
|
async fn connect(daemon: &Daemon) -> arr_db::Db {
|
|
arr_db::Db::connect(daemon.database_path())
|
|
.await
|
|
.expect("connect to the daemon's own database")
|
|
}
|
|
|
|
/// Where DESIGN.md §15 puts a sidecar for `video`, e.g. `<stem>.pt-PT.srt`.
|
|
fn sidecar_path(video: &Path, suffix: &str) -> PathBuf {
|
|
let stem = video
|
|
.file_stem()
|
|
.and_then(|stem| stem.to_str())
|
|
.expect("the seeded video path has a UTF-8 stem");
|
|
video.with_file_name(format!("{stem}.{suffix}"))
|
|
}
|
|
|
|
/// A media file with no subtitles gets one fetched from a provider, synced
|
|
/// by real `alass`, and written under the §15 sidecar name — and once
|
|
/// satisfied, the loop never asks the provider again.
|
|
#[tokio::test]
|
|
async fn a_missing_subtitle_is_fetched_synced_and_named() {
|
|
let podnapisi = MockServer::start().await;
|
|
let pid = "movie-one";
|
|
Mock::given(method("GET"))
|
|
.and(path("/subtitles/search/advanced"))
|
|
.respond_with(
|
|
ResponseTemplate::new(200).set_body_string(
|
|
serde_json::json!({
|
|
"data": [{
|
|
"id": pid,
|
|
"language": "pt-PT",
|
|
"flags": [],
|
|
"releases": ["Movie.One.2024.1080p.WEB-DL-GROUP"],
|
|
"custom_releases": []
|
|
}],
|
|
"page": 1,
|
|
"all_pages": 1
|
|
})
|
|
.to_string(),
|
|
),
|
|
)
|
|
.mount(&podnapisi)
|
|
.await;
|
|
Mock::given(method("GET"))
|
|
.and(path(format!("/subtitles/{pid}/download")))
|
|
.respond_with(
|
|
ResponseTemplate::new(200)
|
|
.set_body_raw(fixtures::PODNAPISI_PLAUSIBLE_ZIP, "application/zip"),
|
|
)
|
|
.mount(&podnapisi)
|
|
.await;
|
|
let podnapisi_url = format!("{}/subtitles", podnapisi.uri());
|
|
|
|
let daemon = Daemon::spawn_with_env(
|
|
UNUSED_UPSTREAM,
|
|
UNUSED_UPSTREAM,
|
|
UNUSED_UPSTREAM,
|
|
&[("ARR_PODNAPISI_URL", &podnapisi_url)],
|
|
)
|
|
.await;
|
|
let database = connect(&daemon).await;
|
|
let (media_file_id, video) = seed_movie_file(
|
|
&database,
|
|
daemon.media_root(),
|
|
900_001,
|
|
"Movie.One.2024.1080p.WEB-DL-GROUP",
|
|
Some(&serde_json::json!({ "sub_tracks": [] })),
|
|
)
|
|
.await;
|
|
configure_subtitles(daemon.base_url(), &["pt-PT"], &["podnapisi"], None).await;
|
|
|
|
let subtitle = wait_for_subtitle(daemon.base_url(), media_file_id, "provider").await;
|
|
assert_eq!(subtitle["language"], "pt-PT");
|
|
assert_eq!(subtitle["provider"], "podnapisi");
|
|
assert_eq!(
|
|
subtitle["sync"], "synced",
|
|
"real alass produced a plausible shift"
|
|
);
|
|
|
|
let expected_path = sidecar_path(&video, "pt-PT.srt");
|
|
assert_eq!(
|
|
subtitle["path"].as_str(),
|
|
Some(expected_path.to_string_lossy()).as_deref()
|
|
);
|
|
assert!(
|
|
expected_path.exists(),
|
|
"sidecar written at the §15 name: {}",
|
|
expected_path.display()
|
|
);
|
|
|
|
// §15: once satisfied, the loop never searches this language again.
|
|
// Wait past one more 30-second reconcile tick and confirm nothing moved.
|
|
tokio::time::sleep(Duration::from_secs(35)).await;
|
|
let requests = podnapisi
|
|
.received_requests()
|
|
.await
|
|
.expect("request recording is on by default");
|
|
assert_eq!(
|
|
requests.len(),
|
|
3,
|
|
"a pt-PT want searches both pt-PT and pt-BR (§15's substitution), plus one \
|
|
download, and never again once satisfied: {requests:?}"
|
|
);
|
|
}
|
|
|
|
/// A file whose only English text lives in an embedded track gets that track
|
|
/// extracted by real `ffmpeg` and translated into a `.mt` sidecar — the
|
|
/// improvement over Bazarr §15 calls out explicitly.
|
|
#[tokio::test]
|
|
async fn an_embedded_track_is_extracted_and_translated() {
|
|
let scripts = tempfile::tempdir().expect("tempdir for the stub translator");
|
|
let template = write_stub_translator(scripts.path());
|
|
|
|
let daemon = Daemon::spawn_with_env(
|
|
UNUSED_UPSTREAM,
|
|
UNUSED_UPSTREAM,
|
|
UNUSED_UPSTREAM,
|
|
&[(
|
|
"ARR_TRANSLATE_COMMAND_TEMPLATE",
|
|
template.to_str().expect("utf8 path"),
|
|
)],
|
|
)
|
|
.await;
|
|
let database = connect(&daemon).await;
|
|
let (media_file_id, video) = seed_movie_file(
|
|
&database,
|
|
daemon.media_root(),
|
|
900_002,
|
|
"Movie.Two.2024.1080p.WEB-DL-GROUP",
|
|
Some(&serde_json::json!({ "sub_tracks": [
|
|
{ "language": "en", "codec": "subrip", "forced": false, "sdh": false }
|
|
] })),
|
|
)
|
|
.await;
|
|
// No provider enabled: §15 translates immediately once nothing else has
|
|
// the language, no waiting window.
|
|
configure_subtitles(daemon.base_url(), &["pt-PT"], &[], Some("command")).await;
|
|
|
|
let translated = wait_for_subtitle(daemon.base_url(), media_file_id, "translated").await;
|
|
assert_eq!(translated["language"], "pt-PT");
|
|
assert_eq!(translated["engine"], "command");
|
|
|
|
let subtitles: Vec<serde_json::Value> = reqwest::get(format!(
|
|
"{}/api/media-files/{media_file_id}/subtitles",
|
|
daemon.base_url()
|
|
))
|
|
.await
|
|
.expect("list subtitles")
|
|
.json()
|
|
.await
|
|
.expect("subtitles json");
|
|
let extracted = subtitles
|
|
.iter()
|
|
.find(|subtitle| subtitle["origin"] == "extracted")
|
|
.expect("the embedded English track was extracted as the translation source");
|
|
assert_eq!(extracted["language"], "en");
|
|
|
|
let sidecar = sidecar_path(&video, "pt-PT.mt.srt");
|
|
let content = tokio::fs::read_to_string(&sidecar)
|
|
.await
|
|
.expect("read the translated sidecar");
|
|
assert!(
|
|
content.contains("Ola"),
|
|
"the stub's translation reached the sidecar: {content}"
|
|
);
|
|
}
|
|
|
|
/// A subtitle whose sync is implausible — real `alass` correlating a
|
|
/// five-minutes-off cue against a three-second clip — keeps the unsynced
|
|
/// original and is flagged, rather than silently writing a bad shift.
|
|
#[tokio::test]
|
|
async fn an_implausible_sync_keeps_the_original_and_is_flagged() {
|
|
let podnapisi = MockServer::start().await;
|
|
let pid = "movie-three";
|
|
Mock::given(method("GET"))
|
|
.and(path("/subtitles/search/advanced"))
|
|
.respond_with(
|
|
ResponseTemplate::new(200).set_body_string(
|
|
serde_json::json!({
|
|
"data": [{
|
|
"id": pid,
|
|
"language": "pt-PT",
|
|
"flags": [],
|
|
"releases": ["Movie.Three.2024.1080p.WEB-DL-GROUP"],
|
|
"custom_releases": []
|
|
}],
|
|
"page": 1,
|
|
"all_pages": 1
|
|
})
|
|
.to_string(),
|
|
),
|
|
)
|
|
.mount(&podnapisi)
|
|
.await;
|
|
Mock::given(method("GET"))
|
|
.and(path(format!("/subtitles/{pid}/download")))
|
|
.respond_with(
|
|
ResponseTemplate::new(200)
|
|
.set_body_raw(fixtures::PODNAPISI_FARFETCHED_ZIP, "application/zip"),
|
|
)
|
|
.mount(&podnapisi)
|
|
.await;
|
|
let podnapisi_url = format!("{}/subtitles", podnapisi.uri());
|
|
|
|
let daemon = Daemon::spawn_with_env(
|
|
UNUSED_UPSTREAM,
|
|
UNUSED_UPSTREAM,
|
|
UNUSED_UPSTREAM,
|
|
&[("ARR_PODNAPISI_URL", &podnapisi_url)],
|
|
)
|
|
.await;
|
|
let database = connect(&daemon).await;
|
|
let (media_file_id, video) = seed_movie_file(
|
|
&database,
|
|
daemon.media_root(),
|
|
900_003,
|
|
"Movie.Three.2024.1080p.WEB-DL-GROUP",
|
|
Some(&serde_json::json!({ "sub_tracks": [] })),
|
|
)
|
|
.await;
|
|
configure_subtitles(daemon.base_url(), &["pt-PT"], &["podnapisi"], None).await;
|
|
|
|
let subtitle = wait_for_subtitle(daemon.base_url(), media_file_id, "provider").await;
|
|
assert_eq!(subtitle["sync"], "rejected");
|
|
|
|
let sidecar = sidecar_path(&video, "pt-PT.srt");
|
|
let content = tokio::fs::read_to_string(&sidecar)
|
|
.await
|
|
.expect("read the unsynced sidecar");
|
|
assert!(
|
|
content.contains("completely unrelated dialogue") && content.contains("00:05:00"),
|
|
"the unsynced original is kept verbatim, timings untouched: {content}"
|
|
);
|
|
|
|
let queue: serde_json::Value =
|
|
reqwest::get(format!("{}/api/queues/subtitles", daemon.base_url()))
|
|
.await
|
|
.expect("get subtitle queue")
|
|
.json()
|
|
.await
|
|
.expect("queue json");
|
|
let movie_gaps = queue["movies"]
|
|
.as_array()
|
|
.expect("movies array")
|
|
.iter()
|
|
.find(|movie| movie["media_file_id"] == media_file_id)
|
|
.expect("the rejected sync surfaces in the missing-subtitles queue (#202)");
|
|
assert!(
|
|
movie_gaps["gaps"]
|
|
.as_array()
|
|
.expect("gaps array")
|
|
.iter()
|
|
.any(|gap| gap["reason"] == "sync_rejected"),
|
|
"{movie_gaps}"
|
|
);
|
|
}
|