//! The subtitle surface (`DESIGN.md` §15, §9.1, §9.3, issue #199). //! //! Everything here is an **operator** action, and that is the whole reason //! this module is separate from the reconcile loop's own subtitle work //! (#196). §15 says manual actions bypass the wanted set: the operator asking //! for a Spanish subtitle gets a Spanish subtitle, whether or not Spanish is //! wanted, and the loop must not then read it as a gap or take it away again. //! That is one line of code — every write here ends by marking the language //! satisfied — but it is the point of the module. //! //! Unlike the release deck, which is asynchronous because Prowlarr fan-out is //! slow and its results are persisted (`releases`), a subtitle search is one //! or two HTTP calls and its candidates are not stored anywhere. So these //! handlers do the work inline and answer with the result, rather than //! returning 202 and leaving the operator to poll. The consequence the client //! has to know about: candidate ids are meaningful only to the provider that //! issued them, and a grab therefore repeats the fact (`sdh`) the search //! reported, because nothing on the server remembers it. A forced candidate //! is never eligible (§15, #185), so a grab has no `forced` flag to accept — //! there is nowhere to put one (issue #233). //! //! Verdict vocabulary is §9.3's, unchanged: `eligible` or `rejected` plus the //! name of the rule that killed it, exactly as `Release` spells it, so the //! manual-search view needs no second concept for subtitles. use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::Arc; use arr_core::subs::{rank, SubtitleTarget, SubtitleVerdict}; use arr_core::{layout, Language, SubtitleCodec}; use arr_db::subtitles as db; use arr_db::SubtitleOrigin; use arr_subs::{CandidateId, MediaFile, MediaRef, SearchRequest}; use axum::extract::rejection::JsonRejection; use axum::extract::{Path as UrlPath, State}; use axum::http::StatusCode; use axum::Json; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use crate::movies::{pool, ApiError, ErrorBody}; use crate::policies::parsed; use crate::state::AppState; use crate::translations::Translation; /// One subtitle arr knows about, as the API renders it. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct Subtitle { pub id: i64, pub media_file_id: i64, /// The tag `arr_core::Language` spells: `pt-PT`, `pt-BR`, `en`. pub language: String, /// `embedded`, `extracted`, `provider` or `translated` (§15). pub origin: String, /// The provider a fetch came from, and its own handle for the candidate. pub provider: Option, pub candidate_id: Option, /// The translation backend, when arr made this one. pub engine: Option, /// Foreign lines and signs only. Never satisfies a want (§15). pub forced: bool, pub sdh: bool, /// `not_run`, `synced` or `rejected` — what `alass` did (§15). A rejected /// sync means the unsynced original was kept and the file is flagged. pub sync: String, /// The sidecar next to the video. `null` only for an embedded track, /// which is inside the container and has no file of its own. pub path: Option, } impl From for Subtitle { fn from(file: arr_db::SubtitleFile) -> Self { Self { id: file.id, media_file_id: file.media_file_id, language: file.language, origin: origin_name(file.origin).to_owned(), provider: file.provider, candidate_id: file.candidate_id, engine: file.engine, forced: file.forced, sdh: file.sdh, sync: sync_name(file.sync).to_owned(), path: file.path, } } } const fn origin_name(origin: SubtitleOrigin) -> &'static str { match origin { SubtitleOrigin::Embedded => "embedded", SubtitleOrigin::Extracted => "extracted", SubtitleOrigin::Provider => "provider", SubtitleOrigin::Translated => "translated", } } const fn sync_name(sync: arr_db::SubtitleSync) -> &'static str { match sync { arr_db::SubtitleSync::NotRun => "not_run", arr_db::SubtitleSync::Synced => "synced", arr_db::SubtitleSync::Rejected => "rejected", } } /// One candidate a provider offered, with the verdict that placed it. /// /// Rejected candidates are in the same list rather than hidden: §9.3's rule /// is that every rejected row names the rule that killed it, so an /// over-strict filter is visible without reading names. // Five independent facts, not a state machine: every combination occurs. #[allow(clippy::struct_excessive_bools)] #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SubtitleCandidate { /// The provider offering it, and its own handle for it. Both travel back /// unchanged in a grab. pub provider: String, pub candidate_id: String, pub language: String, /// Whether the provider matched the exact file by `moviehash` — §15's /// outright winner. pub hash_match: bool, /// Whether the candidate's release name is the one the file was /// imported under — §15's second ranking tier. Reported rather than /// left implicit in the order, because §9.3's manual view shows the /// facts that decided a row, and the release name of the file on disk /// is not otherwise on the wire. pub release_match: bool, pub release_name: Option, pub group: Option, pub source: Option, pub rating: Option, pub download_count: Option, pub forced: bool, pub sdh: bool, /// `eligible` or `rejected`, the same words the release deck uses. pub verdict: String, /// The rule that rejected it, `null` when eligible. pub rejected_rule: Option, } /// One wanted language a media file still lacks, and why (§15, §9.6). /// /// `reason` collapses `subtitle_attempts.state` to the words the title /// detail page shows (issue #201): `searching`, `no_candidates`, `capped` /// or `failed`. A wanted language with no attempt row yet — the reconcile /// loop has not reached it — reads the same as `searching`: no verdict /// exists either way. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct MissingSubtitle { pub language: String, pub reason: String, /// Why the last attempt failed. Set only when `reason` is `failed`. pub detail: Option, } /// An embedded track this file carries that could be written out as a /// sidecar on request (§15, issue #260). /// /// `subtitle_files` records an embedded track by the language it satisfies /// and nothing else, so the codec — the fact that decides whether a track is /// text or bitmaps — lives only in `media_files.probed`. The status endpoint /// reads it from there and joins it in, because the panel that offers the /// extraction is already loading status and must not need a second call to /// find out what it may offer. /// /// Only the extractable ones are listed. An image-format track carries /// bitmaps and arr does not OCR (§15), and a forced track is excluded too: /// §15 gives a language exactly one sidecar and no filename segment marks a /// forced one, so extracting it would spend the language's only slot on the /// signs-only track. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct EmbeddedTrack { /// Position among the file's subtitle streams — what an extract names, /// not the container's absolute stream id. pub index: usize, /// The tag `arr_core::Language` spells, as the probe resolved it. pub language: String, /// `ffprobe`'s own codec name: `subrip`, `ass` or `mov_text`. pub codec: String, /// Complete, with sound descriptions. Satisfies, ranked below plain. pub sdh: bool, } /// One media file's subtitles and the wanted languages it still lacks. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SubtitleStatus { pub media_file_id: i64, /// Ordered by language (§9.6's one row of chips). pub subtitles: Vec, pub missing: Vec, /// Text tracks inside the video that an extract could turn into a /// sidecar (§15, #260). Empty for a file probed before #189, whose /// `sub_tracks` carry a language and no codec. pub embedded_tracks: Vec, /// Manual translations still running, or failed since the request that /// started them returned. pub translations: Vec, } /// One episode's media file, subtitles and gaps — the series-wide bulk /// sibling of [`SubtitleStatus`], joined the same way `series::files` /// already joins episode files, so the series detail page costs one call /// rather than one per episode. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct EpisodeSubtitleStatus { pub episode_id: i64, pub media_file_id: i64, pub subtitles: Vec, pub missing: Vec, pub embedded_tracks: Vec, pub translations: Vec, } /// One wanted language a media file lacks, or a subtitle it has but `alass` /// flagged (§15, issue #202). #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SubtitleGap { pub language: String, /// [`reason_of`]'s words — `no_candidates`, `capped` or `failed` — plus /// `sync_rejected` for a subtitle that exists but whose sync `alass` /// rejected (§15): not a missing language, but still something the /// operator did not see happen. pub reason: String, /// Set only when `reason` is `failed`. pub detail: Option, } /// A movie with at least one subtitle gap, for the missing-subtitles queue. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct MovieSubtitleGaps { pub movie_id: i64, pub tmdb_id: i64, pub title: String, pub year: Option, pub poster_path: Option, pub media_file_id: i64, pub gaps: Vec, } /// One episode's gaps, named for the operator the way [`QueuedEpisode`] is /// (`SxxEyy` comes from season and episode numbers). #[derive(Debug, Clone, Serialize, ToSchema)] pub struct EpisodeSubtitleGaps { pub episode_id: i64, pub media_file_id: i64, pub season_number: i64, pub episode_number: i64, pub gaps: Vec, } /// A season collapsed into one row because every one of its episodes carries /// the identical gap — a season-wide provider or budget failure otherwise /// floods the queue one row per episode, the restraint §9.5 already applies /// to the TV attention queues. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SeasonSubtitleGaps { pub season_number: i64, pub media_file_ids: Vec, pub gaps: Vec, } /// A series with at least one subtitle gap. One row per series (§9.5's /// restraint): episodes that did not collapse into a season stay listed /// individually, the rest roll up into `seasons`. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SeriesSubtitleGaps { pub series_id: i64, pub tmdb_id: i64, pub title: String, pub year: Option, pub poster_path: Option, pub episodes: Vec, pub seasons: Vec, } /// The missing-subtitles queue (issue #202): every title with an unsatisfied /// wanted language and why, plus subtitles a sync rejected. Each entry's /// gaps are resolved with the manual actions this module already offers — /// search, translate, or a retried search. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SubtitleQueue { pub movies: Vec, pub series: Vec, } /// A provider that could not answer this search. /// /// One unreachable provider does not fail the search: §15 configures two at /// once, and the operator can still grab from whichever answered. The failure /// is reported rather than swallowed so "no candidates" and "nobody could be /// asked" stay distinguishable. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SubtitleProviderError { pub provider: String, pub error: String, } /// What one manual search found. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct SubtitleSearchResults { /// Ranked best first, rejected candidates last (§15's ranking, #185). pub candidates: Vec, /// Providers that were asked and could not answer. pub provider_errors: Vec, } /// Which language to search for. #[derive(Debug, Clone, Deserialize, ToSchema)] pub struct SubtitleSearchInput { /// Exactly as `arr_core::Language` spells it. Matched exactly: pt-PT and /// pt-BR are separate searches here, because the operator asked for one /// of them by name and §15's "pt-BR is accepted" is a rule about the /// wanted set, not about a manual request. pub language: String, } /// The candidate to fetch, repeated from the search results. #[derive(Debug, Clone, Deserialize, ToSchema)] pub struct SubtitleGrabInput { pub provider: String, pub candidate_id: String, pub language: String, /// The candidate's own SDH flag, as the search reported it. Nothing on /// the server remembers a search, so it travels with the grab; it is a /// fact about the subtitle and is stored with it. No `forced` flag /// exists here: ranking rejects every forced candidate outright (§15), /// so a grab never has one to record (issue #233). #[serde(default)] pub sdh: bool, } /// Which embedded track to write out as a sidecar (§15, #260). #[derive(Debug, Clone, Deserialize, ToSchema)] pub struct SubtitleExtractInput { /// The `index` of an [`EmbeddedTrack`] the status endpoint listed: /// position among the file's subtitle streams, not the container's /// absolute stream id. pub track_index: usize, } /// The source subtitle, the language wanted, and which engine to use. #[derive(Debug, Clone, Deserialize, ToSchema)] pub struct SubtitleTranslateInput { /// Any subtitle already on this file is a legal source (§15), including /// one extracted from an embedded track and including another machine /// translation. An unextracted embedded track is not: it has no text. pub source_subtitle_id: i64, pub target_language: String, /// Defaults to the configured `translation_engine` when omitted. #[serde(default)] pub engine: Option, } /// Everything one manual action needs to know about the file it targets. struct Target { media_file_id: i64, path: PathBuf, size: u64, media: MediaRef, /// The release the file was imported under, when a grab record still /// says. Ranking scores an exact match against it (§15). release_name: Option, release_group: Option, source: Option, } impl Target { fn search_request(&self, language: Language) -> SearchRequest { SearchRequest { file: MediaFile { path: self.path.clone(), size: self.size, release_name: self.release_name.clone(), media: self.media, }, languages: vec![language], } } /// Where a sidecar for `language` belongs: next to the video, inside the /// §7.4 title folder, named by §15's rule. fn sidecar(&self, language: &Language, machine_translated: bool) -> Result { let name = self .path .file_name() .and_then(std::ffi::OsStr::to_str) .ok_or_else(|| { ApiError::Database(format!( "media file path {} has no name", self.path.display() )) })?; let parent = self.path.parent().ok_or_else(|| { ApiError::Database(format!( "media file path {} has no folder", self.path.display() )) })?; Ok(parent.join(layout::subtitle_name(name, language, machine_translated))) } } /// Load the file and everything ranking needs to score candidates for it. async fn target(state: &AppState, media_file_id: i64) -> Result { let file = sqlx::query!( r#"SELECT id AS "id!: i64", path AS "path!: String", size AS "size!: i64", owner_kind AS "owner_kind!: String", owner_id AS "owner_id!: i64" FROM media_files WHERE id = ?"#, media_file_id ) .fetch_optional(pool(state)?) .await? .ok_or(ApiError::MediaFileNotFound)?; let media = media_ref(state, &file.owner_kind, file.owner_id).await?; let release = imported_release(state, &file.owner_kind, file.owner_id).await?; let (release_name, claims) = match release { Some((name, parsed)) => { let claims: arr_core::ParsedRelease = serde_json::from_value(parsed) .map_err(|error| ApiError::Database(error.to_string()))?; (Some(name), Some(claims)) } None => (None, None), }; Ok(Target { media_file_id: file.id, path: PathBuf::from(file.path), size: u64::try_from(file.size).unwrap_or(0), media, release_name, release_group: claims.as_ref().and_then(|claims| claims.group.clone()), source: claims .as_ref() .and_then(|claims| claims.source) .map(Into::into), }) } /// The TMDB coordinates providers search by (§15, `arr_subs::MediaRef`). async fn media_ref( state: &AppState, owner_kind: &str, owner_id: i64, ) -> Result { if owner_kind == "movie" { let tmdb_id = sqlx::query_scalar!( r#"SELECT tmdb_id AS "tmdb_id!: i64" FROM movies WHERE id = ?"#, owner_id ) .fetch_optional(pool(state)?) .await? .ok_or(ApiError::NotFound)?; return Ok(MediaRef::Movie { tmdb_id: u64::try_from(tmdb_id).unwrap_or(0), }); } let row = sqlx::query!( r#"SELECT sr.tmdb_id AS "tmdb_id!: i64", s.number AS "season!: i64", e.number AS "episode!: i64" FROM episodes e JOIN seasons s ON s.id = e.season_id JOIN series sr ON sr.id = s.series_id WHERE e.id = ?"#, owner_id ) .fetch_optional(pool(state)?) .await? .ok_or(ApiError::EpisodeNotFound)?; Ok(MediaRef::Episode { tmdb_id: u64::try_from(row.tmdb_id).unwrap_or(0), season: u16::try_from(row.season).unwrap_or(0), episode: u16::try_from(row.episode).unwrap_or(0), }) } /// The release a file was imported under, best effort. /// /// An episode that arrived inside a season pack has no grab of its own, so /// the season's grab is the fallback. Nothing here is load-bearing: a missing /// release name costs the exact-name tier in ranking and nothing else. async fn imported_release( state: &AppState, owner_kind: &str, owner_id: i64, ) -> Result, ApiError> { let own = sqlx::query!( r#"SELECT r.name AS "name!: String", r.parsed AS "parsed!: serde_json::Value" FROM grabs g JOIN releases r ON r.id = g.release_id WHERE g.target_kind = ? AND g.target_id = ? ORDER BY g.imported_at DESC, g.id DESC LIMIT 1"#, owner_kind, owner_id ) .fetch_optional(pool(state)?) .await?; if let Some(row) = own { return Ok(Some((row.name, row.parsed))); } if owner_kind != "episode" { return Ok(None); } let pack = sqlx::query!( r#"SELECT r.name AS "name!: String", r.parsed AS "parsed!: serde_json::Value" FROM episodes e JOIN grabs g ON g.target_kind = 'season' AND g.target_id = e.season_id JOIN releases r ON r.id = g.release_id WHERE e.id = ? ORDER BY g.imported_at DESC, g.id DESC LIMIT 1"#, owner_id ) .fetch_optional(pool(state)?) .await?; Ok(pack.map(|row| (row.name, row.parsed))) } /// The file's own `moviehash`, when it can be computed. /// /// Best effort: the video may be on a mount that is temporarily gone, and a /// subtitle search that cannot hash still ranks — it just loses §15's /// outright winner. Reading the head and tail of a large file is blocking IO, /// so it does not run on the async worker. async fn moviehash(path: &Path, size: u64) -> Option { let owned = path.to_path_buf(); let computed = tokio::task::spawn_blocking(move || arr_subs::moviehash(&owned, size)) .await .ok()?; match computed { Ok(hash) => hash, Err(error) => { tracing::warn!(path = %path.display(), %error, "moviehash not computed"); None } } } /// Which providers a manual search runs: those this deployment has /// credentials for, intersected with `providers_enabled` (§15). /// /// A provider the operator switched off in `/settings` is not asked, because /// "enabled" is the operator's own statement about which sources to use. A /// grab does not go through here — naming a candidate is a stronger statement /// than the setting, and the candidate came from somewhere. async fn enabled_providers(state: &AppState) -> Result>, ApiError> { let raw = sqlx::query_scalar!( r#"SELECT providers_enabled AS "providers_enabled!: String" FROM subtitle_settings WHERE id = 1"# ) .fetch_one(pool(state)?) .await?; let enabled: BTreeSet = serde_json::from_str(&raw).map_err(|error| ApiError::Database(error.to_string()))?; Ok(state .subtitle_providers() .iter() .filter(|provider| enabled.contains(provider.id().as_str())) .map(Arc::clone) .collect()) } /// A language tag as the domain spells it. fn language_of(tag: &str) -> Result { if tag.trim().is_empty() { return Err(ApiError::Invalid("language: must not be empty".into())); } Ok(arr_db::policy::language(tag)) } /// Whether two release names are the same one, matched exactly as /// `arr_core::subs`' ranking tier does — case-insensitively, and never when /// either side is unknown. fn same_release(candidate: Option<&str>, target: Option<&str>) -> bool { match (candidate, target) { (Some(candidate), Some(target)) => candidate.eq_ignore_ascii_case(target), _ => false, } } async fn subtitles_of(state: &AppState, media_file_id: i64) -> Result, ApiError> { let files = db::files_for(pool(state)?, media_file_id).await?; Ok(files.into_iter().map(Subtitle::from).collect()) } /// Every subtitle on every file one owner has, ordered by file then language. async fn subtitles_for_owner( state: &AppState, owner_kind: &str, owner_id: i64, ) -> Result, ApiError> { let ids = sqlx::query_scalar!( r#"SELECT id AS "id!: i64" FROM media_files WHERE owner_kind = ? AND owner_id = ? ORDER BY path"#, owner_kind, owner_id ) .fetch_all(pool(state)?) .await?; let mut out = Vec::new(); for id in ids { out.extend(subtitles_of(state, id).await?); } Ok(out) } /// The global wanted set (§15), as `/settings` last saved it. async fn wanted_languages(state: &AppState) -> Result, ApiError> { let raw = sqlx::query_scalar!( r#"SELECT wanted_languages AS "wanted_languages!: String" FROM subtitle_settings WHERE id = 1"# ) .fetch_one(pool(state)?) .await?; serde_json::from_str(&raw).map_err(|error| ApiError::Database(error.to_string())) } /// The extractable text tracks one media file carries (§15, #260). /// /// Parsed as leniently as the reconcile loop parses the same column: a row /// written before #189 carries a language and no codec, and must read as /// "nothing extractable here" rather than fail the whole status call. That /// is the same set of files the issue puts out of scope — they are /// re-downloaded, not migrated. async fn embedded_tracks_of( state: &AppState, media_file_id: i64, ) -> Result, ApiError> { let probed = sqlx::query_scalar!( r#"SELECT probed FROM media_files WHERE id = ?"#, media_file_id ) .fetch_optional(pool(state)?) .await? .flatten(); Ok(probed.as_deref().map_or_default(extractable_tracks)) } /// The `sub_tracks` array of a `probed` blob, filtered to what §15 allows an /// extraction to touch. fn extractable_tracks(probed: &str) -> Vec { let Ok(value) = serde_json::from_str::(probed) else { return Vec::new(); }; let Some(tracks) = value .get("sub_tracks") .and_then(serde_json::Value::as_array) else { return Vec::new(); }; tracks .iter() .enumerate() .filter_map(|(index, track)| { let language = track.get("language")?.as_str()?.to_owned(); let codec = track.get("codec")?.as_str()?.to_owned(); if !SubtitleCodec::from_probe_name(&codec).is_text() { return None; } let flag = |name: &str| { track .get(name) .and_then(serde_json::Value::as_bool) .unwrap_or(false) }; if flag("forced") { return None; } Some(EmbeddedTrack { index, language, codec, sdh: flag("sdh"), }) }) .collect() } /// `subtitle_attempts.state` in the title detail page's own words. fn reason_of(attempt: &arr_db::SubtitleAttempt) -> (String, Option) { match attempt.state { // Satisfied never reaches here: a satisfied language is dropped // before this is called (satisfaction is read off the files, not // the attempt row — DESIGN.md §15). arr_db::SubtitleState::Wanted | arr_db::SubtitleState::Satisfied => { ("searching".to_owned(), None) } arr_db::SubtitleState::Unavailable => ("no_candidates".to_owned(), None), // Two budgets and a provider 429 all write `capped`; the sentence is // the only thing that says which. arr_db::SubtitleState::Capped => ("capped".to_owned(), attempt.last_failure.clone()), arr_db::SubtitleState::Failed => ("failed".to_owned(), attempt.last_failure.clone()), } } /// The wanted languages one media file still lacks, with why. /// /// Satisfaction is decided here against the files actually on this file, /// never against `subtitle_attempts.state` — that column is the loop's own /// bookkeeping and DESIGN.md §15 is explicit that it must never become a /// second source of truth for what counts as satisfied. async fn missing_for( state: &AppState, media_file_id: i64, subtitles: &[Subtitle], wanted: &[String], ) -> Result, ApiError> { let gaps: Vec<&String> = wanted .iter() .filter(|language| { !subtitles.iter().any(|subtitle| { !subtitle.forced && arr_core::subs::satisfies(language, &subtitle.language) }) }) .collect(); if gaps.is_empty() { return Ok(Vec::new()); } let attempts = db::attempts_for(pool(state)?, media_file_id).await?; Ok(gaps .into_iter() .map(|language| { let (reason, detail) = attempts .iter() .find(|attempt| attempt.language == *language) .map_or_else(|| ("searching".to_owned(), None), reason_of); MissingSubtitle { language: language.clone(), reason, detail, } }) .collect()) } /// Every media file one owner has, with its subtitles and its gaps (§9.6). async fn status_for_owner( state: &AppState, owner_kind: &str, owner_id: i64, ) -> Result, ApiError> { let ids = sqlx::query_scalar!( r#"SELECT id AS "id!: i64" FROM media_files WHERE owner_kind = ? AND owner_id = ? ORDER BY path"#, owner_kind, owner_id ) .fetch_all(pool(state)?) .await?; let wanted = wanted_languages(state).await?; let mut out = Vec::with_capacity(ids.len()); for id in ids { let subtitles = subtitles_of(state, id).await?; let missing = missing_for(state, id, &subtitles, &wanted).await?; out.push(SubtitleStatus { media_file_id: id, subtitles, missing, embedded_tracks: embedded_tracks_of(state, id).await?, translations: state.translations().of(id), }); } Ok(out) } #[utoipa::path( get, path = "/api/movies/{movie_id}/subtitles/status", tag = "subtitles", params(("movie_id" = i64, Path, description = "Movie row id")), responses( (status = 200, body = [SubtitleStatus]), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn status_for_movie( State(state): State, UrlPath(movie_id): UrlPath, ) -> Result>, ApiError> { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM movies WHERE id = ?) AS "exists!: bool""#, movie_id ) .fetch_one(pool(&state)?) .await?; if !exists { return Err(ApiError::NotFound); } Ok(Json(status_for_owner(&state, "movie", movie_id).await?)) } #[utoipa::path( get, path = "/api/episodes/{episode_id}/subtitles/status", tag = "subtitles", params(("episode_id" = i64, Path, description = "Episode row id")), responses( (status = 200, body = [SubtitleStatus]), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn status_for_episode( State(state): State, UrlPath(episode_id): UrlPath, ) -> Result>, ApiError> { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM episodes WHERE id = ?) AS "exists!: bool""#, episode_id ) .fetch_one(pool(&state)?) .await?; if !exists { return Err(ApiError::EpisodeNotFound); } Ok(Json(status_for_owner(&state, "episode", episode_id).await?)) } #[utoipa::path( get, path = "/api/series/{series_id}/subtitles/status", tag = "subtitles", params(("series_id" = i64, Path, description = "Series row id")), responses( (status = 200, body = [EpisodeSubtitleStatus]), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn status_for_series( State(state): State, UrlPath(series_id): UrlPath, ) -> Result>, ApiError> { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM series WHERE id = ?) AS "exists!: bool""#, series_id ) .fetch_one(pool(&state)?) .await?; if !exists { return Err(ApiError::NotFound); } let rows = sqlx::query!( r#"SELECT mf.id AS "media_file_id!: i64", e.id AS "episode_id!: i64" FROM media_files mf JOIN episodes e ON mf.owner_kind = 'episode' AND e.id = mf.owner_id JOIN seasons se ON se.id = e.season_id WHERE se.series_id = ? ORDER BY mf.path"#, series_id ) .fetch_all(pool(&state)?) .await?; let wanted = wanted_languages(&state).await?; let mut out = Vec::with_capacity(rows.len()); for row in rows { let subtitles = subtitles_of(&state, row.media_file_id).await?; let missing = missing_for(&state, row.media_file_id, &subtitles, &wanted).await?; out.push(EpisodeSubtitleStatus { episode_id: row.episode_id, media_file_id: row.media_file_id, subtitles, missing, embedded_tracks: embedded_tracks_of(&state, row.media_file_id).await?, translations: state.translations().of(row.media_file_id), }); } Ok(Json(out)) } #[utoipa::path( get, path = "/api/media-files/{media_file_id}/subtitles", tag = "subtitles", params(("media_file_id" = i64, Path, description = "Media file row id")), responses( (status = 200, body = [Subtitle]), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn list_for_media_file( State(state): State, UrlPath(media_file_id): UrlPath, ) -> Result>, ApiError> { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM media_files WHERE id = ?) AS "exists!: bool""#, media_file_id ) .fetch_one(pool(&state)?) .await?; if !exists { return Err(ApiError::MediaFileNotFound); } Ok(Json(subtitles_of(&state, media_file_id).await?)) } #[utoipa::path( get, path = "/api/movies/{movie_id}/subtitles", tag = "subtitles", params(("movie_id" = i64, Path, description = "Movie row id")), responses( (status = 200, body = [Subtitle]), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn list_for_movie( State(state): State, UrlPath(movie_id): UrlPath, ) -> Result>, ApiError> { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM movies WHERE id = ?) AS "exists!: bool""#, movie_id ) .fetch_one(pool(&state)?) .await?; if !exists { return Err(ApiError::NotFound); } Ok(Json(subtitles_for_owner(&state, "movie", movie_id).await?)) } #[utoipa::path( get, path = "/api/episodes/{episode_id}/subtitles", tag = "subtitles", params(("episode_id" = i64, Path, description = "Episode row id")), responses( (status = 200, body = [Subtitle]), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn list_for_episode( State(state): State, UrlPath(episode_id): UrlPath, ) -> Result>, ApiError> { let exists = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM episodes WHERE id = ?) AS "exists!: bool""#, episode_id ) .fetch_one(pool(&state)?) .await?; if !exists { return Err(ApiError::EpisodeNotFound); } Ok(Json( subtitles_for_owner(&state, "episode", episode_id).await?, )) } #[utoipa::path( post, path = "/api/media-files/{media_file_id}/subtitles/search", tag = "subtitles", params(("media_file_id" = i64, Path, description = "Media file row id")), request_body = SubtitleSearchInput, responses( (status = 200, body = SubtitleSearchResults), (status = 404, body = ErrorBody), (status = 422, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn search( State(state): State, UrlPath(media_file_id): UrlPath, body: Result, JsonRejection>, ) -> Result, ApiError> { let input = parsed(body)?; let language = language_of(&input.language)?; let target = target(&state, media_file_id).await?; let providers = enabled_providers(&state).await?; if providers.is_empty() { return Err(ApiError::SubtitleUpstream( "no subtitle provider is both configured and enabled".into(), )); } let request = target.search_request(language.clone()); let mut offered = Vec::new(); let mut provider_errors = Vec::new(); for provider in &providers { match provider.search(&request).await { Ok(candidates) => { offered.extend(candidates.into_iter().filter(|c| c.language == language)); } Err(error) => provider_errors.push(SubtitleProviderError { provider: provider.id().to_string(), error: error.to_string(), }), } } let hash = moviehash(&target.path, target.size).await; let ranking_target = SubtitleTarget { moviehash: hash.as_deref(), release_name: target.release_name.as_deref(), release_group: target.release_group.as_deref(), source: target.source, }; let cores: Vec<_> = offered .iter() .map(|candidate| candidate.to_core(hash.as_deref())) .collect(); let candidates = rank(&ranking_target, &cores) .into_iter() .map(|ranked| { let candidate = &offered[ranked.index]; let (verdict, rejected_rule) = match ranked.verdict { SubtitleVerdict::Eligible => ("eligible", None), SubtitleVerdict::Rejected(rule) => ("rejected", Some(rule.name().to_owned())), }; SubtitleCandidate { provider: candidate.provider.to_string(), candidate_id: candidate.id.to_string(), language: candidate.language.to_string(), hash_match: candidate.hash_match, release_match: same_release( candidate.release_name.as_deref(), target.release_name.as_deref(), ), release_name: candidate.release_name.clone(), group: candidate.group.clone(), source: candidate.source.map(|source| source.to_string()), rating: candidate.rating, download_count: candidate.download_count, forced: candidate.forced, sdh: candidate.sdh, verdict: verdict.to_owned(), rejected_rule, } }) .collect(); Ok(Json(SubtitleSearchResults { candidates, provider_errors, })) } #[utoipa::path( post, path = "/api/media-files/{media_file_id}/subtitles/grab", tag = "subtitles", params(("media_file_id" = i64, Path, description = "Media file row id")), request_body = SubtitleGrabInput, responses( (status = 201, body = Subtitle), (status = 404, body = ErrorBody), (status = 409, body = ErrorBody), (status = 422, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn grab( State(state): State, UrlPath(media_file_id): UrlPath, body: Result, JsonRejection>, ) -> Result<(StatusCode, Json), ApiError> { let input = parsed(body)?; let language = language_of(&input.language)?; let target = target(&state, media_file_id).await?; let provider = state .subtitle_provider(&input.provider) .ok_or_else(|| { ApiError::Invalid(format!( "provider: '{}' is not configured on this deployment", input.provider )) })? .clone(); let destination = target.sidecar(&language, false)?; claim_language(&state, target.media_file_id, &language).await?; claim_path(&state, &destination).await?; let fetched = provider .download(&CandidateId::new(input.candidate_id.clone())) .await .map_err(|error| match error { arr_subs::Error::NotFound { .. } => ApiError::SubtitleCandidateExpired, other => ApiError::SubtitleUpstream(other.to_string()), })?; let text = srt_text(&fetched)?; write_sidecar(&destination, &text).await?; let sync = state.syncer().settle(&target.path, &destination).await; if let Some(synced) = &sync.content { write_sidecar(&destination, synced).await?; } let mut record = arr_db::NewSubtitleFile::fetched( target.media_file_id, &language.to_string(), &input.provider, &input.candidate_id, &destination.to_string_lossy(), ) .sync(db_sync_state(sync.state)); if input.sdh { record = record.sdh(); } finish(&state, &record, target.media_file_id, &language).await } #[utoipa::path( post, path = "/api/media-files/{media_file_id}/subtitles/extract", tag = "subtitles", params(("media_file_id" = i64, Path, description = "Media file row id")), request_body = SubtitleExtractInput, responses( (status = 201, body = Subtitle), (status = 404, body = ErrorBody), (status = 409, body = ErrorBody), (status = 422, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] /// Write one embedded text track out as a sidecar SRT (§15, issue #260). /// /// The reconcile loop extracts only when it is already translating a gap, so /// on a file where nothing is missing no sidecar is ever written and the /// panel's translate lane has no source to offer. This is the operator's own /// way in, and it obeys §15's rule that a manual action bypasses the wanted /// set: the language is marked satisfied whether or not it was wanted. /// /// `alass` is deliberately not run. §15 syncs every *fetched* and every /// *translated* subtitle; a track lifted out of the container already /// carries the video's own timings, and shifting it against itself could /// only make it worse. pub async fn extract( State(state): State, UrlPath(media_file_id): UrlPath, body: Result, JsonRejection>, ) -> Result<(StatusCode, Json), ApiError> { let input = parsed(body)?; let target = target(&state, media_file_id).await?; let track = embedded_tracks_of(&state, media_file_id) .await? .into_iter() .find(|track| track.index == input.track_index) .ok_or_else(|| { ApiError::Invalid(format!( "track_index: this file has no extractable text track at {}", input.track_index )) })?; let language = language_of(&track.language)?; let destination = target.sidecar(&language, false)?; claim_language(&state, target.media_file_id, &language).await?; claim_path(&state, &destination).await?; state .extractor() .extract_srt(&target.path, track.index, &destination) .await .map_err(|error| { ApiError::Filesystem(format!( "extracting stream {} of {}: {error}", track.index, target.path.display() )) })?; let mut record = arr_db::NewSubtitleFile::extracted( target.media_file_id, &track.language, &destination.to_string_lossy(), ); if track.sdh { record = record.sdh(); } finish(&state, &record, target.media_file_id, &language).await } #[utoipa::path( post, path = "/api/media-files/{media_file_id}/subtitles/translate", tag = "subtitles", params(("media_file_id" = i64, Path, description = "Media file row id")), request_body = SubtitleTranslateInput, responses( (status = 202, body = Translation), (status = 404, body = ErrorBody), (status = 409, body = ErrorBody), (status = 422, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] /// Check the request, then translate as a task of its own and answer at /// once. The run reports on the file's subtitle status: `translations` /// while it runs or once it fails, the sidecar on `subtitles` once it lands. pub async fn translate( State(state): State, UrlPath(media_file_id): UrlPath, body: Result, JsonRejection>, ) -> Result<(StatusCode, Json), ApiError> { let input = parsed(body)?; let target_language = language_of(&input.target_language)?; let target = target(&state, media_file_id).await?; let source = db::files_for(pool(&state)?, media_file_id) .await? .into_iter() .find(|file| file.id == input.source_subtitle_id) .ok_or(ApiError::SubtitleNotFound)?; let source_path = source.path.clone().ok_or_else(|| { ApiError::Invalid( "source_subtitle_id: an embedded track carries no text; extract it first".into(), ) })?; let source_language = language_of(&source.language)?; if source_language == target_language { return Err(ApiError::Invalid( "target_language: same as the source subtitle's language".into(), )); } let engine = engine_name(&state, input.engine.as_deref()).await?; let backend = state .translation_backend(&engine) .ok_or_else(|| ApiError::SubtitleUpstream(arr_subs::unusable_engine(&engine)))? .clone(); let destination = target.sidecar(&target_language, true)?; claim_language(&state, target.media_file_id, &target_language).await?; claim_path(&state, &destination).await?; let tag = target_language.to_string(); let run = state .translations() .start(media_file_id, &tag, &engine) .ok_or_else(|| ApiError::Conflict(format!("{tag} is already being translated")))?; tokio::spawn(async move { let job = TranslationJob { target, source_path, source_language, target_language, destination, engine, backend, }; match job.run(&state).await { Ok(()) => state.translations().finish(media_file_id, &tag), Err(detail) => { tracing::warn!(media_file_id, language = %tag, %detail, "manual translation failed"); state.translations().fail(media_file_id, &tag, detail); } } }); Ok((StatusCode::ACCEPTED, Json(run))) } /// Everything a manual translation needs once the request has been checked. struct TranslationJob { target: Target, source_path: String, source_language: Language, target_language: Language, destination: PathBuf, engine: String, backend: Arc, } impl TranslationJob { /// The reason it failed, in words for the panel. async fn run(self, state: &AppState) -> Result<(), String> { let raw = tokio::fs::read_to_string(&self.source_path) .await .map_err(|error| format!("{}: {error}", self.source_path))?; let cues = arr_subs::srt::parse(&raw) .map_err(|error| format!("source subtitle is not SRT: {error}"))?; let translated = arr_subs::translate::translate( self.backend.as_ref(), &cues, &self.source_language, &self.target_language, ) .await .map_err(|error| error.to_string())?; write_sidecar(&self.destination, &arr_subs::srt::render(&translated)) .await .map_err(failure)?; let sync = state .syncer() .settle(&self.target.path, &self.destination) .await; if let Some(synced) = &sync.content { write_sidecar(&self.destination, synced) .await .map_err(failure)?; } let record = arr_db::NewSubtitleFile::translated( self.target.media_file_id, &self.target_language.to_string(), &self.engine, &self.destination.to_string_lossy(), ) .sync(db_sync_state(sync.state)); finish( state, &record, self.target.media_file_id, &self.target_language, ) .await .map(drop) .map_err(failure) } } /// An [`ApiError`] from the write half of a translation, which has no /// response left to carry it. fn failure(error: ApiError) -> String { match error { ApiError::Filesystem(detail) | ApiError::Conflict(detail) => detail, ApiError::Database(detail) => format!("database error: {detail}"), other => format!("{other:?}"), } } #[utoipa::path( delete, path = "/api/subtitles/{subtitle_id}", tag = "subtitles", params(("subtitle_id" = i64, Path, description = "Subtitle row id")), responses( (status = 204), (status = 404, body = ErrorBody), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn delete( State(state): State, UrlPath(subtitle_id): UrlPath, ) -> Result { let row = sqlx::query!( r#"SELECT media_file_id AS "media_file_id!: i64", language AS "language!: String", path FROM subtitle_files WHERE id = ?"#, subtitle_id ) .fetch_optional(pool(&state)?) .await? .ok_or(ApiError::SubtitleNotFound)?; if let Some(path) = &row.path { match tokio::fs::remove_file(path).await { Ok(()) => {} // Already gone is the outcome asked for. Anything else is a real // failure and the row stays, so a retry still has something to // delete rather than leaving an orphan sidecar behind. Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(ApiError::Filesystem(format!("{path}: {error}"))), } } db::delete_file(pool(&state)?, subtitle_id).await?; // §15 reads satisfaction off the files, so a language with nothing left // is a gap again and the loop must be able to see it. let remaining = sqlx::query_scalar!( r#"SELECT EXISTS( SELECT 1 FROM subtitle_files WHERE media_file_id = ? AND language = ? AND forced = 0 ) AS "exists!: bool""#, row.media_file_id, row.language ) .fetch_one(pool(&state)?) .await?; if !remaining { db::unsatisfy(pool(&state)?, row.media_file_id, &row.language).await?; } Ok(StatusCode::NO_CONTENT) } /// The engine to translate with: the one asked for, else the configured one. async fn engine_name(state: &AppState, requested: Option<&str>) -> Result { if let Some(engine) = requested { if !arr_subs::ENGINES.contains(&engine) { return Err(ApiError::Invalid(format!( "engine: '{engine}' is not a known engine" ))); } return Ok(engine.to_owned()); } sqlx::query_scalar!( r#"SELECT translation_engine AS "translation_engine: String" FROM subtitle_settings WHERE id = 1"# ) .fetch_one(pool(state)?) .await? .ok_or_else(|| ApiError::Invalid("engine: no translation engine is configured".into())) } /// Refuse a second subtitle for a language this file already has (#222). /// /// §15 gives a language exactly one sidecar and no upgrade loop: replacing /// one is delete-then-fetch, which the panel offers as a button (#223). The /// schema carries the same invariant as a unique index, so this only decides /// *when* the refusal happens — before the download and the write, rather /// than on the insert with a stray file already on disk. async fn claim_language( state: &AppState, media_file_id: i64, language: &Language, ) -> Result<(), ApiError> { let tag = language.to_string(); if let Some(existing) = db::sidecar_for(pool(state)?, media_file_id, &tag).await? { return Err(ApiError::Conflict(format!( "{tag} already has a subtitle at {existing}; delete it first to replace it" ))); } Ok(()) } /// Refuse to write over a sidecar arr already knows about. /// /// [`claim_language`] covers the common case, and this still covers the rest: /// the filename is the row's identity on disk, and two `media_files` rows for /// one video — a re-import that left a stale row behind — derive the same /// sidecar name from different ids. Two rows never race for one name. async fn claim_path(state: &AppState, destination: &Path) -> Result<(), ApiError> { let path = destination.to_string_lossy().into_owned(); let taken = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM subtitle_files WHERE path = ?) AS "exists!: bool""#, path ) .fetch_one(pool(state)?) .await?; if taken { return Err(ApiError::Conflict(format!( "a subtitle already exists at {path}; delete it first" ))); } Ok(()) } /// A fetched subtitle as SRT text. /// /// Sidecars are SRT (§15) and converting other container formats is its own /// issue (#213), so anything else is refused rather than written under a /// `.srt` name it does not honour. fn srt_text(fetched: &arr_subs::Fetched) -> Result { // Sidecars are SRT (§15), but a provider serving VTT or ASS is converted // rather than refused (#213). Decoding happens inside `to_srt`, and a // format with no parser still fails here rather than reaching the disk. fetched .to_srt() .map_err(|error| ApiError::SubtitleUpstream(error.to_string())) } /// Map `arr_subs`'s three-state sync result onto the column pair `arr_db` /// stores it as. `Syncer::settle` already folded an unusable `alass` and an /// implausible result together into "nothing changed" — this is just the /// vocabulary switch between the crate that ran `alass` and the one that /// persists what it decided. const fn db_sync_state(state: arr_subs::SyncState) -> arr_db::SubtitleSync { match state { arr_subs::SyncState::NotRun => arr_db::SubtitleSync::NotRun, arr_subs::SyncState::Synced => arr_db::SubtitleSync::Synced, arr_subs::SyncState::Rejected => arr_db::SubtitleSync::Rejected, } } /// Write a sidecar whole or not at all, so Jellyfin never reads a half file. async fn write_sidecar(destination: &Path, text: &str) -> Result<(), ApiError> { let failure = |path: &Path, error: std::io::Error| { ApiError::Filesystem(format!("{}: {error}", path.display())) }; let temp = destination.with_extension("srt.partial"); tokio::fs::write(&temp, text) .await .map_err(|error| failure(&temp, error))?; if let Err(error) = tokio::fs::rename(&temp, destination).await { let _ = tokio::fs::remove_file(&temp).await; return Err(failure(destination, error)); } Ok(()) } /// Record a written sidecar and settle the language it answers. /// /// The `mark_satisfied` is §15's "manual actions bypass the wanted-set /// logic": the loop stops working on that language whether or not it was in /// the wanted set, so a manually requested Spanish subtitle is never treated /// as a gap and never replaced. Every grab reaching this point is a plain or /// SDH subtitle — a forced candidate is never eligible, so there is no /// exception left to carve out of the satisfaction claim (issue #233). async fn finish( state: &AppState, record: &arr_db::NewSubtitleFile, media_file_id: i64, language: &Language, ) -> Result<(StatusCode, Json), ApiError> { let id = db::record_file(pool(state)?, record).await?; db::mark_satisfied(pool(state)?, media_file_id, &language.to_string()).await?; let subtitle = db::files_for(pool(state)?, media_file_id) .await? .into_iter() .find(|file| file.id == id) .ok_or(ApiError::SubtitleNotFound)?; refresh_jellyfin(state).await; Ok((StatusCode::CREATED, Json(Subtitle::from(subtitle)))) } /// Ask Jellyfin to rescan, the same single call §7.5 already makes on /// import. Its filesystem watcher misses a sidecar dropped in next to a file /// it already knows about, and a failure here must not fail the write that /// already landed on disk. async fn refresh_jellyfin(state: &AppState) { let Some(jellyfin) = state.jellyfin() else { return; }; if let Err(error) = jellyfin.refresh().await { tracing::warn!(%error, "jellyfin refresh failed"); } } #[utoipa::path( get, path = "/api/queues/subtitles", tag = "subtitles", responses( (status = 200, body = SubtitleQueue), (status = 500, body = ErrorBody), (status = 503, body = ErrorBody) ) )] pub async fn queue(State(state): State) -> Result, ApiError> { // A language dropped from the wanted set (§15) is not a gap any more — // its attempt row just has not been cleaned up yet. `missing_for` (#201) // already applies this bound; the queue reads the same way. let wanted: BTreeSet = wanted_languages(&state).await?.into_iter().collect(); Ok(Json(SubtitleQueue { movies: movie_gaps(&state, &wanted).await?, series: series_gaps(&state, &wanted).await?, })) } /// `subtitle_attempts.state` in [`SubtitleGap`]'s words — the same vocabulary /// [`reason_of`] uses for the title detail page (#201), so a gap reads the /// same wherever it is shown. fn attempt_reason( state: arr_db::SubtitleState, last_failure: Option, ) -> (String, Option) { match state { arr_db::SubtitleState::Wanted | arr_db::SubtitleState::Satisfied => { ("searching".to_owned(), None) } arr_db::SubtitleState::Unavailable => ("no_candidates".to_owned(), None), arr_db::SubtitleState::Capped => ("capped".to_owned(), last_failure), arr_db::SubtitleState::Failed => ("failed".to_owned(), last_failure), } } async fn movie_gaps( state: &AppState, wanted: &BTreeSet, ) -> Result, ApiError> { let database = pool(state)?; let mut out: Vec = Vec::new(); let attempts = sqlx::query!( r#"SELECT m.id AS "movie_id!: i64", m.tmdb_id AS "tmdb_id!: i64", m.title AS "title!: String", m.year, m.poster_path, mf.id AS "media_file_id!: i64", sa.language AS "language!: String", sa.state AS "state!: arr_db::SubtitleState", sa.last_failure FROM subtitle_attempts sa JOIN media_files mf ON mf.id = sa.media_file_id AND mf.owner_kind = 'movie' JOIN movies m ON m.id = mf.owner_id WHERE sa.state IN ('failed', 'capped', 'unavailable') ORDER BY m.title, sa.language"# ) .fetch_all(database) .await?; for row in attempts { if !wanted.contains(&row.language) { continue; } let (reason, detail) = attempt_reason(row.state, row.last_failure); push_movie_gap( &mut out, row.movie_id, row.tmdb_id, &row.title, row.year, row.poster_path, row.media_file_id, SubtitleGap { language: row.language, reason, detail, }, ); } let rejected = sqlx::query!( r#"SELECT m.id AS "movie_id!: i64", m.tmdb_id AS "tmdb_id!: i64", m.title AS "title!: String", m.year, m.poster_path, mf.id AS "media_file_id!: i64", sf.language AS "language!: String" FROM subtitle_files sf JOIN media_files mf ON mf.id = sf.media_file_id AND mf.owner_kind = 'movie' JOIN movies m ON m.id = mf.owner_id WHERE sf.sync_rejected = 1 ORDER BY m.title, sf.language"# ) .fetch_all(database) .await?; for row in rejected { push_movie_gap( &mut out, row.movie_id, row.tmdb_id, &row.title, row.year, row.poster_path, row.media_file_id, SubtitleGap { language: row.language, reason: "sync_rejected".to_owned(), detail: None, }, ); } Ok(out) } #[allow(clippy::too_many_arguments)] fn push_movie_gap( entries: &mut Vec, movie_id: i64, tmdb_id: i64, title: &str, year: Option, poster_path: Option, media_file_id: i64, gap: SubtitleGap, ) { if let Some(entry) = entries.iter_mut().find(|entry| entry.movie_id == movie_id) { entry.gaps.push(gap); return; } entries.push(MovieSubtitleGaps { movie_id, tmdb_id, title: title.to_owned(), year, poster_path, media_file_id, gaps: vec![gap], }); } async fn series_gaps( state: &AppState, wanted: &BTreeSet, ) -> Result, ApiError> { let database = pool(state)?; let mut out: Vec = Vec::new(); let attempts = sqlx::query!( r#"SELECT s.id AS "series_id!: i64", s.tmdb_id AS "tmdb_id!: i64", s.title AS "title!: String", s.year, s.poster_path, e.id AS "episode_id!: i64", se.number AS "season_number!: i64", e.number AS "episode_number!: i64", mf.id AS "media_file_id!: i64", sa.language AS "language!: String", sa.state AS "state!: arr_db::SubtitleState", sa.last_failure FROM subtitle_attempts sa JOIN media_files mf ON mf.id = sa.media_file_id AND mf.owner_kind = 'episode' JOIN episodes e ON e.id = mf.owner_id JOIN seasons se ON se.id = e.season_id JOIN series s ON s.id = se.series_id WHERE sa.state IN ('failed', 'capped', 'unavailable') ORDER BY s.title, se.number, e.number, sa.language"# ) .fetch_all(database) .await?; for row in attempts { if !wanted.contains(&row.language) { continue; } let (reason, detail) = attempt_reason(row.state, row.last_failure); push_episode_gap( &mut out, row.series_id, row.tmdb_id, &row.title, row.year, row.poster_path, row.episode_id, row.season_number, row.episode_number, row.media_file_id, SubtitleGap { language: row.language, reason, detail, }, ); } let rejected = sqlx::query!( r#"SELECT s.id AS "series_id!: i64", s.tmdb_id AS "tmdb_id!: i64", s.title AS "title!: String", s.year, s.poster_path, e.id AS "episode_id!: i64", se.number AS "season_number!: i64", e.number AS "episode_number!: i64", mf.id AS "media_file_id!: i64", sf.language AS "language!: String" FROM subtitle_files sf JOIN media_files mf ON mf.id = sf.media_file_id AND mf.owner_kind = 'episode' JOIN episodes e ON e.id = mf.owner_id JOIN seasons se ON se.id = e.season_id JOIN series s ON s.id = se.series_id WHERE sf.sync_rejected = 1 ORDER BY s.title, se.number, e.number, sf.language"# ) .fetch_all(database) .await?; for row in rejected { push_episode_gap( &mut out, row.series_id, row.tmdb_id, &row.title, row.year, row.poster_path, row.episode_id, row.season_number, row.episode_number, row.media_file_id, SubtitleGap { language: row.language, reason: "sync_rejected".to_owned(), detail: None, }, ); } for series in &mut out { collapse_seasons(series); } Ok(out) } #[allow(clippy::too_many_arguments)] fn push_episode_gap( entries: &mut Vec, series_id: i64, tmdb_id: i64, title: &str, year: Option, poster_path: Option, episode_id: i64, season_number: i64, episode_number: i64, media_file_id: i64, gap: SubtitleGap, ) { if !entries.iter().any(|entry| entry.series_id == series_id) { entries.push(SeriesSubtitleGaps { series_id, tmdb_id, title: title.to_owned(), year, poster_path, episodes: Vec::new(), seasons: Vec::new(), }); } let entry = entries .iter_mut() .find(|entry| entry.series_id == series_id) .unwrap_or_else(|| unreachable!()); match entry .episodes .iter_mut() .find(|episode| episode.episode_id == episode_id) { Some(episode) => episode.gaps.push(gap), None => entry.episodes.push(EpisodeSubtitleGaps { episode_id, media_file_id, season_number, episode_number, gaps: vec![gap], }), } } /// Roll a season's episodes into one [`SeasonSubtitleGaps`] row when two or /// more of them carry the identical set of gaps — the season-wide failure /// §9.5's restraint is meant to catch. A season with only one gapped episode, /// or episodes whose gaps differ, stays as individual episode rows. fn collapse_seasons(series: &mut SeriesSubtitleGaps) { let mut by_group: std::collections::BTreeMap<(i64, String), Vec> = std::collections::BTreeMap::new(); for episode in std::mem::take(&mut series.episodes) { let key = (episode.season_number, gap_signature(&episode.gaps)); by_group.entry(key).or_default().push(episode); } for ((season_number, _signature), mut episodes) in by_group { if let [first, ..] = episodes.as_slice() { if episodes.len() >= 2 { series.seasons.push(SeasonSubtitleGaps { season_number, media_file_ids: episodes .iter() .map(|episode| episode.media_file_id) .collect(), gaps: first.gaps.clone(), }); continue; } } series.episodes.append(&mut episodes); } series .episodes .sort_by_key(|episode| (episode.season_number, episode.episode_number)); series.seasons.sort_by_key(|season| season.season_number); } fn gap_signature(gaps: &[SubtitleGap]) -> String { let mut parts: Vec = gaps .iter() .map(|gap| { format!( "{}|{}|{}", gap.language, gap.reason, gap.detail.as_deref().unwrap_or("") ) }) .collect(); parts.sort(); parts.join(",") } #[cfg(test)] #[allow(clippy::too_many_lines)] mod tests { use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; use std::sync::Arc; use arr_core::Language; use arr_subs::{ Backend, BackendId, Batch, Candidate, CandidateId, DownloadFuture, Fetched, Provider, ProviderId, SearchFuture, SearchRequest, SubtitleFormat, TranslateFuture, TranslatedCue, }; use axum::http::StatusCode; use tokio::io::AsyncWriteExt; use crate::{router, AppState, Upstreams}; const SRT: &str = "1\n00:00:01,000 --> 00:00:02,000\nolá\n"; /// §15's second ranking tier, as the manual view reads it: the same /// release name matches whatever its casing, and an unknown name on /// either side is never a match. #[test] fn release_names_match_case_insensitively_and_never_when_unknown() { assert!(super::same_release( Some("Dune.2021.2160p.WEB-DL-GROUP"), Some("dune.2021.2160p.web-dl-group") )); assert!(!super::same_release(Some("Dune.2021"), Some("Dune.2024"))); assert!(!super::same_release(Some("Dune.2021"), None)); assert!(!super::same_release(None, Some("Dune.2021"))); assert!(!super::same_release(None, None)); } /// Offers three candidates for whatever it is asked: one plain, one that /// matched by hash, one forced. #[derive(Debug)] struct StubProvider { id: ProviderId, format: SubtitleFormat, body: String, } impl StubProvider { fn new(name: &str) -> Self { Self { id: ProviderId::new(name), format: SubtitleFormat::Srt, body: SRT.to_owned(), } } fn serving(name: &str, format: SubtitleFormat) -> Self { Self { id: ProviderId::new(name), format, body: SRT.to_owned(), } } /// Serve a format together with a body actually in that format, so a /// conversion test exercises the parser rather than the error path. fn serving_body(name: &str, format: SubtitleFormat, body: &str) -> Self { Self { id: ProviderId::new(name), format, body: body.to_owned(), } } fn candidate(&self, id: &str, language: &Language) -> Candidate { Candidate { provider: self.id.clone(), id: CandidateId::new(id), language: language.clone(), hash_match: false, release_name: None, group: None, source: None, rating: Some(5.0), download_count: Some(10), forced: false, sdh: false, } } } impl Provider for StubProvider { fn id(&self) -> ProviderId { self.id.clone() } fn search<'a>(&'a self, request: &'a SearchRequest) -> SearchFuture<'a> { Box::pin(async move { let language = request.languages[0].clone(); Ok(vec![ self.candidate("plain", &language), Candidate { hash_match: true, ..self.candidate("hashed", &language) }, Candidate { forced: true, ..self.candidate("forced", &language) }, // A language nobody asked for: providers may answer with // more than they were asked, and ranking discards it. self.candidate("other-language", &Language::Other("fr".into())), ]) }) } fn download<'a>(&'a self, id: &'a CandidateId) -> DownloadFuture<'a> { Box::pin(async move { Ok(Fetched { id: id.clone(), language: Language::PortuguesePortugal, format: self.format.clone(), content: self.body.as_bytes().to_vec(), }) }) } fn probe(&self) -> arr_subs::ProbeFuture<'_> { Box::pin(async move { Ok(()) }) } } /// A provider that is configured but never answers. #[derive(Debug)] struct DeadProvider; impl Provider for DeadProvider { fn id(&self) -> ProviderId { ProviderId::new("dead") } fn search<'a>(&'a self, _request: &'a SearchRequest) -> SearchFuture<'a> { Box::pin(async move { Err(arr_subs::Error::Unauthorized { provider: ProviderId::new("dead"), }) }) } fn download<'a>(&'a self, id: &'a CandidateId) -> DownloadFuture<'a> { Box::pin(async move { Err(arr_subs::Error::NotFound { provider: ProviderId::new("dead"), candidate: id.clone(), }) }) } fn probe(&self) -> arr_subs::ProbeFuture<'_> { Box::pin(async move { Err(arr_subs::Error::Unauthorized { provider: ProviderId::new("dead"), }) }) } } /// Uppercases every cue. Enough to prove the pipeline, and it keeps cue /// numbering intact so `translate`'s validation passes. #[derive(Debug)] struct StubBackend; impl Backend for StubBackend { fn id(&self) -> BackendId { BackendId::new("openai") } fn supports(&self, _target: &Language) -> bool { true } fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> { Box::pin(async move { Ok(batch .cues .iter() .map(|cue| TranslatedCue { number: cue.number, text: cue.text.to_uppercase(), }) .collect()) }) } fn probe(&self) -> arr_subs::translate::ProbeFuture<'_> { Box::pin(async move { Ok(()) }) } } struct Fixture { _dir: tempfile::TempDir, base: String, pool: sqlx::SqlitePool, media_file_id: i64, folder: PathBuf, video: PathBuf, } impl Fixture { async fn subtitle_rows(&self) -> Vec { arr_db::subtitles::files_for(&self.pool, self.media_file_id) .await .expect("files") } /// Give the file a `probed` column, the way import writes one. async fn probe(&self, sub_tracks: serde_json::Value) { let probed = serde_json::json!({ "resolution": "1080p", "source": null, "hdr": "SDR", "audio_tracks": [], "sub_tracks": sub_tracks, }) .to_string(); sqlx::query("UPDATE media_files SET probed = ? WHERE id = ?") .bind(probed) .bind(self.media_file_id) .execute(&self.pool) .await .expect("probe"); } /// Put a real container behind the `media_files` row: three `subrip` /// tracks, `eng` / `por` / `eng`, so a real `ffmpeg` has something to /// extract and the stream index means something. async fn real_video(&self) { let clip = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("../arr-probe/tests/fixtures/movie-1080p.mkv"); tokio::fs::copy(&clip, &self.video).await.expect("clip"); } async fn attempt_state(&self, language: &str) -> Option { sqlx::query_scalar::<_, String>( "SELECT state FROM subtitle_attempts WHERE media_file_id = ? AND language = ?", ) .bind(self.media_file_id) .bind(language) .fetch_optional(&self.pool) .await .expect("attempt") } } async fn application( providers: Vec>, backends: Vec>, ) -> Fixture { build_fixture(providers, backends, arr_subs::Syncer::default()).await } async fn application_with_syncer( providers: Vec>, backends: Vec>, syncer: arr_subs::Syncer, ) -> Fixture { build_fixture(providers, backends, syncer).await } async fn build_fixture( providers: Vec>, backends: Vec>, syncer: arr_subs::Syncer, ) -> Fixture { let dir = tempfile::tempdir().expect("tempdir"); let database = arr_db::Db::connect(dir.path().join("arr.db")) .await .expect("connect database"); database.migrate().await.expect("migrate database"); let pool = database.pool().clone(); // §7.4: one folder per title, the sidecar lands inside it. let folder = dir.path().join("Dune (2021) [tmdbid-438631]"); tokio::fs::create_dir_all(&folder).await.expect("folder"); let video = folder.join("Dune (2021) [tmdbid-438631] - [2160p][WEB-DL].mkv"); // Big enough for a `moviehash`: OpenSubtitles hashes the first and // last 64 KiB, and a shorter file has no hash at all — which would // silently drop §15's outright-winning ranking tier from the tests. tokio::fs::write(&video, vec![7u8; 200_000]) .await .expect("video"); sqlx::query( "INSERT INTO movies (id, tmdb_id, title, year, root_id) VALUES (1, 438631, 'Dune', 2021, 1)", ) .execute(&pool) .await .expect("movie"); let path = video.to_string_lossy().into_owned(); sqlx::query( "INSERT INTO media_files (id, owner_kind, owner_id, path, size) VALUES (1, 'movie', 1, ?, 200000)", ) .bind(&path) .execute(&pool) .await .expect("media file"); let state = AppState::new(Upstreams::new( "http://127.0.0.1:1".into(), "http://127.0.0.1:1".into(), )) .expect("state") .with_database(database) .with_subtitle_providers(providers) .with_translation_backends(backends) .with_syncer(syncer); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind"); let address = listener.local_addr().expect("address"); tokio::spawn(async move { axum::serve(listener, router(state)).await.expect("serve") }); Fixture { _dir: dir, base: format!("http://{address}"), pool, media_file_id: 1, folder, video, } } async fn stub_application() -> Fixture { application(vec![Arc::new(StubProvider::new("opensubtitles"))], vec![]).await } /// Same fixture as [`application`], with a Jellyfin client attached so a /// write can be observed asking it to refresh (§7.5, §15). async fn application_with_jellyfin( providers: Vec>, backends: Vec>, jellyfin_url: &str, ) -> Fixture { let dir = tempfile::tempdir().expect("tempdir"); let database = arr_db::Db::connect(dir.path().join("arr.db")) .await .expect("connect database"); database.migrate().await.expect("migrate database"); let pool = database.pool().clone(); let folder = dir.path().join("Dune (2021) [tmdbid-438631]"); tokio::fs::create_dir_all(&folder).await.expect("folder"); let video = folder.join("Dune (2021) [tmdbid-438631] - [2160p][WEB-DL].mkv"); tokio::fs::write(&video, vec![7u8; 200_000]) .await .expect("video"); sqlx::query( "INSERT INTO movies (id, tmdb_id, title, year, root_id) VALUES (1, 438631, 'Dune', 2021, 1)", ) .execute(&pool) .await .expect("movie"); let path = video.to_string_lossy().into_owned(); sqlx::query( "INSERT INTO media_files (id, owner_kind, owner_id, path, size) VALUES (1, 'movie', 1, ?, 200000)", ) .bind(&path) .execute(&pool) .await .expect("media file"); let jellyfin = crate::jellyfin::JellyfinClient::new(jellyfin_url, None).expect("jellyfin client"); let state = AppState::new(Upstreams::new( "http://127.0.0.1:1".into(), "http://127.0.0.1:1".into(), )) .expect("state") .with_database(database) .with_subtitle_providers(providers) .with_translation_backends(backends) .with_jellyfin(jellyfin); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind"); let address = listener.local_addr().expect("address"); tokio::spawn(async move { axum::serve(listener, router(state)).await.expect("serve") }); Fixture { _dir: dir, base: format!("http://{address}"), pool, media_file_id: 1, folder, video, } } async fn search(fixture: &Fixture, language: &str) -> (StatusCode, serde_json::Value) { let response = reqwest::Client::new() .post(format!( "{}/api/media-files/1/subtitles/search", fixture.base )) .json(&serde_json::json!({ "language": language })) .send() .await .expect("search"); let status = response.status(); (status, response.json().await.expect("search json")) } async fn grab(fixture: &Fixture, body: serde_json::Value) -> (StatusCode, serde_json::Value) { let response = reqwest::Client::new() .post(format!("{}/api/media-files/1/subtitles/grab", fixture.base)) .json(&body) .send() .await .expect("grab"); let status = response.status(); (status, response.json().await.expect("grab json")) } /// Start a translation and wait for it the way the panel does, by /// polling the title's subtitle status. An accepted run answers with the /// subtitle it wrote, or with the run itself once it failed. async fn translate( fixture: &Fixture, body: serde_json::Value, ) -> (StatusCode, serde_json::Value) { let client = reqwest::Client::new(); let target = body["target_language"].clone(); let response = client .post(format!( "{}/api/media-files/1/subtitles/translate", fixture.base )) .json(&body) .send() .await .expect("translate"); let status = response.status(); let started: serde_json::Value = response.json().await.expect("translate json"); if status != StatusCode::ACCEPTED { return (status, started); } for _ in 0..200 { let files: serde_json::Value = client .get(format!("{}/api/movies/1/subtitles/status", fixture.base)) .send() .await .expect("status") .json() .await .expect("status json"); let file = &files[0]; let runs = file["translations"].as_array().expect("translations"); if let Some(run) = runs.iter().find(|run| run["language"] == target) { if run["state"] == "failed" { return (status, run.clone()); } } else if let Some(written) = file["subtitles"] .as_array() .expect("subtitles") .iter() .find(|subtitle| subtitle["language"] == target) { return (status, written.clone()); } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } panic!("translation into {target} never settled"); } fn pt() -> serde_json::Value { serde_json::json!({ "provider": "opensubtitles", "candidate_id": "hashed", "language": "pt-PT" }) } /// §9.3 applied to subtitles: eligible first, every rejected row naming /// the rule that killed it. #[tokio::test] async fn a_search_ranks_candidates_and_names_the_rule_that_rejected_each() { let fixture = stub_application().await; let (status, body) = search(&fixture, "pt-PT").await; assert_eq!(status, StatusCode::OK); let candidates = body["candidates"].as_array().expect("candidates"); // The French candidate the provider volunteered is not in the answer. assert_eq!(candidates.len(), 3, "{body}"); assert_eq!(candidates[0]["candidate_id"], "hashed"); assert_eq!(candidates[0]["verdict"], "eligible"); assert!(candidates[0]["rejected_rule"].is_null()); assert_eq!(candidates[1]["candidate_id"], "plain"); assert_eq!(candidates[2]["candidate_id"], "forced"); assert_eq!(candidates[2]["verdict"], "rejected"); assert_eq!(candidates[2]["rejected_rule"], "forced"); // No grab record backs this file, so nothing can claim its release // name — the chip reads "no", never "unknown". assert_eq!(candidates[0]["release_match"], false); assert_eq!( body["provider_errors"] .as_array() .expect("errors") .as_slice(), [] as [serde_json::Value; 0] ); } /// One provider failing is not the search failing: §15 configures two. #[tokio::test] async fn a_provider_that_cannot_answer_is_reported_beside_the_candidates() { let fixture = application( vec![ Arc::new(StubProvider::new("opensubtitles")), Arc::new(DeadProvider), ], vec![], ) .await; sqlx::query("UPDATE subtitle_settings SET providers_enabled = '[\"opensubtitles\",\"dead\"]' WHERE id = 1") .execute(&fixture.pool) .await .expect("enable both"); let (status, body) = search(&fixture, "pt-PT").await; assert_eq!(status, StatusCode::OK); assert_eq!(body["candidates"].as_array().expect("candidates").len(), 3); let errors = body["provider_errors"].as_array().expect("errors"); assert_eq!(errors.len(), 1, "{body}"); assert_eq!(errors[0]["provider"], "dead"); } /// `providers_enabled` is the operator's statement about which sources /// to use, so a search never reaches a provider left out of it. #[tokio::test] async fn a_disabled_provider_is_never_asked() { let fixture = stub_application().await; sqlx::query("UPDATE subtitle_settings SET providers_enabled = '[]' WHERE id = 1") .execute(&fixture.pool) .await .expect("disable everything"); let (status, _) = search(&fixture, "pt-PT").await; assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn a_search_for_an_unknown_media_file_is_a_404() { let fixture = stub_application().await; let response = reqwest::Client::new() .post(format!( "{}/api/media-files/99/subtitles/search", fixture.base )) .json(&serde_json::json!({ "language": "pt-PT" })) .send() .await .expect("search"); assert_eq!(response.status(), StatusCode::NOT_FOUND); let body: serde_json::Value = response.json().await.expect("json"); assert_eq!(body["error"], "media file not found"); } /// §7.5 applied to subtitles: a grab calls Jellyfin's refresh, the same /// single call import already makes, because its watcher misses a /// sidecar dropped next to a file it already knows about. #[tokio::test] async fn a_grab_refreshes_jellyfin() { let jellyfin = wiremock::MockServer::start().await; wiremock::Mock::given(wiremock::matchers::method("POST")) .and(wiremock::matchers::path("/Library/Refresh")) .respond_with(wiremock::ResponseTemplate::new(204)) .mount(&jellyfin) .await; let fixture = application_with_jellyfin( vec![Arc::new(StubProvider::new("opensubtitles"))], vec![], &jellyfin.uri(), ) .await; let (status, body) = grab(&fixture, pt()).await; assert_eq!(status, StatusCode::CREATED, "{body}"); assert_eq!( jellyfin.received_requests().await.expect("requests").len(), 1 ); } /// A refresh failure must not fail the grab that already landed the /// sidecar on disk (§7.5). #[tokio::test] async fn an_unreachable_jellyfin_does_not_fail_the_grab() { let fixture = application_with_jellyfin( vec![Arc::new(StubProvider::new("opensubtitles"))], vec![], "http://127.0.0.1:1", ) .await; let (status, body) = grab(&fixture, pt()).await; assert_eq!(status, StatusCode::CREATED, "{body}"); } /// §15's disk rule: the sidecar sits next to the video, inside the §7.4 /// folder, named `