diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8f8a202..ec5f3b6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -19,9 +19,12 @@ jobs: - name: Build deps run: | + # ffmpeg is here for ffprobe: arr-probe's tests run it against the + # committed fixture clips (DESIGN.md §12). apt-get update apt-get install -y --no-install-recommends \ - git curl ca-certificates build-essential pkg-config + git curl ca-certificates build-essential pkg-config \ + ffmpeg - name: Cache cargo uses: actions/cache@v4 diff --git a/Cargo.lock b/Cargo.lock index 6b60ad5..4591012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,14 @@ dependencies = [ [[package]] name = "arr-probe" version = "0.1.0" +dependencies = [ + "arr-core", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", +] [[package]] name = "assert-json-diff" diff --git a/crates/arr-probe/Cargo.toml b/crates/arr-probe/Cargo.toml index d1e1a75..c06fbfb 100644 --- a/crates/arr-probe/Cargo.toml +++ b/crates/arr-probe/Cargo.toml @@ -7,6 +7,15 @@ repository.workspace = true publish = false [dependencies] +arr-core.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [lints] workspace = true diff --git a/crates/arr-probe/src/error.rs b/crates/arr-probe/src/error.rs new file mode 100644 index 0000000..d1bc4c8 --- /dev/null +++ b/crates/arr-probe/src/error.rs @@ -0,0 +1,118 @@ +//! Errors the `ffprobe` wrapper can produce. + +use std::path::PathBuf; + +/// Result alias for every fallible operation in this crate. +pub type Result = std::result::Result; + +/// Everything that can go wrong probing a file. +/// +/// Nothing here has a default. A file `ffprobe` cannot read is an error, never +/// a `ProbedFile` full of zeroes — importing on top of a default-valued struct +/// would silently record a 0-byte SDR file with no audio and pass every policy +/// rule that inspects a field it does not have (DESIGN.md §5.6, §5.7). +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// `ffprobe` could not be started at all: not installed, or not on `PATH`. + #[error("could not run {binary}")] + Spawn { + /// The binary that was attempted. + binary: String, + /// The underlying spawn failure. + #[source] + source: std::io::Error, + }, + + /// `ffprobe` started but exited non-zero — an unreadable or corrupt file. + #[error("ffprobe rejected {path}: {stderr}")] + Rejected { + /// The file that was probed. + path: PathBuf, + /// Exit status, absent when the process was killed by a signal. + status: Option, + /// `ffprobe`'s own diagnostics, truncated to something loggable. + stderr: String, + }, + + /// `ffprobe` did not finish within the configured timeout. + #[error("ffprobe timed out probing {path}")] + Timeout { + /// The file that was probed. + path: PathBuf, + }, + + /// Reading the child process failed after it was spawned. + #[error("reading ffprobe output for {path} failed")] + Io { + /// The file that was probed. + path: PathBuf, + /// The underlying IO failure. + #[source] + source: std::io::Error, + }, + + /// `ffprobe` printed something that is not the JSON shape expected. + #[error("ffprobe output for {path} did not parse")] + Decode { + /// The file that was probed. + path: PathBuf, + /// The deserialisation failure. + #[source] + source: serde_json::Error, + }, + + /// `ffprobe` read the file, but as a still image or as text. + /// + /// ffmpeg's `tty` demuxer parses any text file at all as ANSI art video, + /// and the image demuxers turn a poster into a one-frame film. Both would + /// otherwise pass for importable media. + #[error("{path} is not a video file: ffprobe read it as {format}")] + NotVideo { + /// The file that was probed. + path: PathBuf, + /// The container format `ffprobe` matched. + format: String, + }, + + /// `ffprobe` read the file but it carries no video stream. + #[error("{path} has no video stream")] + NoVideoStream { + /// The file that was probed. + path: PathBuf, + }, + + /// A container that reports neither a duration nor a stream long enough to + /// derive one. The runtime sanity check has nothing to work with. + #[error("{path} reports no duration")] + NoDuration { + /// The file that was probed. + path: PathBuf, + }, + + /// Feature selection was handed a set of paths with no readable video file + /// among them. + #[error("no video file among the candidates")] + NoCandidates, +} + +impl Error { + /// Whether this failure is a fact about one file rather than about the + /// prober. + /// + /// Feature selection walks a torrent full of `.nfo`, `.jpg` and `.txt` + /// files, so a file that is not importable media is expected and gets + /// skipped. A missing binary, a timeout or unparseable output is not about + /// the file at all, and swallowing it would report "no video file" for a + /// torrent that has one. + #[must_use] + pub const fn is_about_the_file(&self) -> bool { + matches!( + self, + Self::Rejected { .. } + | Self::NotVideo { .. } + | Self::NoVideoStream { .. } + | Self::NoDuration { .. } + ) + } +} diff --git a/crates/arr-probe/src/ffprobe.rs b/crates/arr-probe/src/ffprobe.rs new file mode 100644 index 0000000..8de0b46 --- /dev/null +++ b/crates/arr-probe/src/ffprobe.rs @@ -0,0 +1,88 @@ +//! The slice of `ffprobe`'s JSON output this crate reads. +//! +//! Everything is optional because `ffprobe` omits fields rather than nulling +//! them, and which ones it omits depends on the container. + +use std::collections::BTreeMap; + +use serde::Deserialize; + +/// One `ffprobe -print_format json -show_format -show_streams` document. +#[derive(Debug, Deserialize)] +pub(crate) struct Output { + #[serde(default)] + pub format: Format, + #[serde(default)] + pub streams: Vec, +} + +/// Container-level facts. +#[derive(Debug, Default, Deserialize)] +pub(crate) struct Format { + #[serde(rename = "format_name")] + pub name: Option, + pub duration: Option, + pub size: Option, +} + +/// One stream of any kind. `codec_type` says which. +#[derive(Debug, Deserialize)] +pub(crate) struct Stream { + pub codec_type: Option, + pub codec_name: Option, + pub width: Option, + pub height: Option, + pub color_transfer: Option, + pub duration: Option, + #[serde(default)] + pub disposition: Disposition, + #[serde(default)] + pub tags: Tags, + #[serde(default)] + pub side_data_list: Vec, +} + +impl Stream { + pub fn is_kind(&self, kind: &str) -> bool { + self.codec_type.as_deref() == Some(kind) + } + + /// A real video stream, not the cover art bolted onto an audio file. + pub fn is_video(&self) -> bool { + self.is_kind("video") && self.disposition.attached_pic == 0 + } +} + +/// Stream flags. Only `attached_pic` is read: an MP3 or FLAC with embedded +/// artwork reports it as a video stream, and taking that for video would make +/// an audio file look importable. +#[derive(Debug, Default, Deserialize)] +pub(crate) struct Disposition { + #[serde(default)] + pub attached_pic: u8, +} + +/// Stream metadata. Matroska writes `HANDLER_NAME`, MP4 writes `handler_name`, +/// so every lookup here is case-insensitive. +#[derive(Debug, Default, Deserialize)] +pub(crate) struct Tags(BTreeMap); + +impl Tags { + pub fn get(&self, key: &str) -> Option<&str> { + self.0 + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(key)) + .map(|(_, value)| value.as_str()) + .filter(|value| !value.is_empty()) + } +} + +/// A stream side-data block. Only the Dolby Vision configuration record and +/// the HDR10+ marker are read; the rest are ignored. +#[derive(Debug, Deserialize)] +pub(crate) struct SideData { + #[serde(rename = "side_data_type")] + pub kind: Option, + pub dv_profile: Option, + pub dv_bl_signal_compatibility_id: Option, +} diff --git a/crates/arr-probe/src/language.rs b/crates/arr-probe/src/language.rs new file mode 100644 index 0000000..fa00bd7 --- /dev/null +++ b/crates/arr-probe/src/language.rs @@ -0,0 +1,145 @@ +//! Turning a track's language tag into an [`arr_core::Language`]. +//! +//! The Portuguese case is not decided here. `arr-core::lang` owns the §5.2 +//! signal order, and this module's job is to hand it the two signals a file +//! carries — the stream `title` and `handler_name` strings, and the +//! container's BCP-47 tag. The third signal, release-name markers, belongs to +//! the release and is deliberately not consulted from a file probe. + +use arr_core::{ + lang::{resolve_portuguese, PortugueseEvidence}, + Language, +}; + +/// ISO-639-2 to ISO-639-1, for the codes `ffprobe` actually emits. +/// +/// TMDB reports a title's original language as ISO-639-1 (`en`), `ffprobe` +/// reports a track's as ISO-639-2/B (`eng`). The §5.2 rule compares the two, +/// so they have to be spelled the same way by the time they meet. +const ISO_639_2_TO_1: [(&str, &str); 40] = [ + ("ara", "ar"), + ("bul", "bg"), + ("cat", "ca"), + ("ces", "cs"), + ("chi", "zh"), + ("cze", "cs"), + ("dan", "da"), + ("deu", "de"), + ("dut", "nl"), + ("ell", "el"), + ("eng", "en"), + ("fin", "fi"), + ("fra", "fr"), + ("fre", "fr"), + ("ger", "de"), + ("gle", "ga"), + ("heb", "he"), + ("hin", "hi"), + ("hun", "hu"), + ("ind", "id"), + ("isl", "is"), + ("ita", "it"), + ("jpn", "ja"), + ("kor", "ko"), + ("nld", "nl"), + ("nor", "no"), + ("pol", "pl"), + ("ron", "ro"), + ("rum", "ro"), + ("rus", "ru"), + ("slk", "sk"), + ("slo", "sk"), + ("spa", "es"), + ("swe", "sv"), + ("tha", "th"), + ("tur", "tr"), + ("ukr", "uk"), + ("und", "und"), + ("vie", "vi"), + ("zho", "zh"), +]; + +/// Resolve one track's language from its tag and its self-description. +pub(crate) fn resolve(tag: Option<&str>, title: Option<&str>, handler: Option<&str>) -> Language { + let normalised = tag.unwrap_or("und").trim().to_ascii_lowercase(); + let primary = normalised + .split(['-', '_']) + .next() + .unwrap_or(normalised.as_str()); + + if matches!(primary, "pt" | "por") { + return resolve_portuguese(PortugueseEvidence { + // Signal 1 is a property of the release, not of the file. + name_markers: &[], + stream_title: title, + handler_name: handler, + container_tag: tag, + }); + } + + let code = ISO_639_2_TO_1 + .iter() + .find(|(long, _)| *long == primary) + .map_or(primary, |(_, short)| *short); + Language::Other(code.to_owned()) +} + +#[cfg(test)] +mod tests { + use arr_core::Language; + + use super::resolve; + + #[test] + fn iso_639_2_becomes_iso_639_1() { + assert_eq!( + resolve(Some("eng"), None, None), + Language::Other("en".to_owned()) + ); + assert_eq!( + resolve(Some("ger"), None, None), + Language::Other("de".to_owned()) + ); + } + + #[test] + fn an_untagged_track_is_undetermined_not_portuguese() { + assert_eq!(resolve(None, None, None), Language::Other("und".to_owned())); + } + + #[test] + fn bcp_47_region_decides_on_its_own() { + assert_eq!( + resolve(Some("pt-BR"), None, None), + Language::PortugueseBrazil + ); + assert_eq!( + resolve(Some("pt-PT"), None, None), + Language::PortuguesePortugal + ); + } + + #[test] + fn the_track_title_decides_when_the_code_cannot() { + assert_eq!( + resolve(Some("por"), Some("Portuguese (Brazil)"), None), + Language::PortugueseBrazil + ); + assert_eq!( + resolve(Some("por"), None, Some("Português (Portugal)")), + Language::PortuguesePortugal + ); + } + + #[test] + fn plain_portuguese_stays_unverified() { + assert_eq!( + resolve(Some("por"), Some("Português"), Some("SoundHandler")), + Language::PortugueseUnverified + ); + assert_eq!( + resolve(Some("pt"), None, None), + Language::PortugueseUnverified + ); + } +} diff --git a/crates/arr-probe/src/lib.rs b/crates/arr-probe/src/lib.rs index cd2adf6..42cd143 100644 --- a/crates/arr-probe/src/lib.rs +++ b/crates/arr-probe/src/lib.rs @@ -1 +1,251 @@ -//! arr-probe — see DESIGN.md. +//! `ffprobe` wrapper — the second phase of truth, DESIGN.md §5.6. +//! +//! Release names lie or omit; files do not. Everything here reads a file that +//! is already on disk and reports what is actually in it: resolution, video +//! codec, HDR format with the Dolby Vision profile as a number, per-track audio +//! languages with the strings the tracks call themselves, subtitle tracks, +//! duration and size. +//! +//! Two things this crate deliberately does not do. It never reports a source +//! type — no file knows whether it came off a disc or a streaming service, so +//! that stays a claim of the release name. And it never returns a +//! default-valued [`ProbedFile`]: a file `ffprobe` cannot read is an +//! [`Error`], because a zeroed struct would sail through every policy rule +//! that inspects a field the file never had. + +use std::{ + ffi::OsString, + path::{Path, PathBuf}, + process::Stdio, + time::Duration, +}; + +use tokio::process::Command; + +pub mod error; +mod ffprobe; +mod language; +mod model; + +pub use error::{Error, Result}; +pub use model::{FeatureSelection, ProbedFile, RuntimeMatch}; + +/// The binary invoked when nothing else is configured. +pub const DEFAULT_BINARY: &str = "ffprobe"; + +/// How long one probe may take. Generous: `ffprobe` on a 60 GB remux over a +/// network mount is slow, and killing it early would fail an import that was +/// about to succeed. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); + +/// How far a candidate's playing time may sit from the expected runtime and +/// still be the feature. Wide enough for an extended cut, narrow enough that a +/// 90-minute extras reel does not pass for a 120-minute film. +pub const RUNTIME_TOLERANCE: f64 = 0.15; + +/// How much of `ffprobe`'s stderr is kept in an error. +const STDERR_LIMIT: usize = 512; + +/// Runs `ffprobe` and turns its output into domain types. +#[derive(Clone, Debug)] +pub struct Prober { + binary: OsString, + timeout: Duration, +} + +impl Default for Prober { + fn default() -> Self { + Self { + binary: DEFAULT_BINARY.into(), + timeout: DEFAULT_TIMEOUT, + } + } +} + +impl Prober { + /// A prober that runs `ffprobe` from `PATH`. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Run a specific binary instead of whatever `PATH` resolves to. + #[must_use] + pub fn with_binary(mut self, binary: impl Into) -> Self { + self.binary = binary.into(); + self + } + + /// Change how long one probe may take. + #[must_use] + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + /// Probe one file. + /// + /// # Errors + /// + /// [`Error::Spawn`] when `ffprobe` is not installed, [`Error::Rejected`] + /// when `ffprobe` cannot read the file, [`Error::Timeout`], + /// [`Error::Decode`] on unexpected output, and + /// [`Error::NoVideoStream`] / [`Error::NoDuration`] when the file is + /// readable but is not a video file this project can import. + pub async fn probe(&self, path: impl Into) -> Result { + let path = path.into(); + let output = self.run(&path).await?; + let document = serde_json::from_slice(&output).map_err(|source| Error::Decode { + path: path.clone(), + source, + })?; + let mut file = ProbedFile::from_output(path, &document)?; + if file.size == 0 { + // Every container ffprobe knows reports a size, but a zero would + // score as a 0-byte release downstream, so never trust it. + file.size = tokio::fs::metadata(&file.path) + .await + .map_err(|source| Error::Io { + path: file.path.clone(), + source, + })? + .len(); + } + Ok(file) + } + + /// Decide which file in a multi-file torrent is the feature. + /// + /// The largest video file wins, but only after a runtime sanity check + /// against `expected_runtime` — usually TMDB's — so that a bulky extras + /// reel does not get imported as the film. When no candidate is within + /// [`RUNTIME_TOLERANCE`], the largest is still returned and the result is + /// marked [`RuntimeMatch::Mismatched`]; deciding what to do about that is + /// the import's call, not this crate's. + /// + /// Paths that are not importable media are skipped rather than failing the + /// selection — a torrent is full of `.nfo` and `.jpg` files. A failure of + /// the prober itself is not skipped: see [`Error::is_about_the_file`]. + /// + /// # Errors + /// + /// [`Error::NoCandidates`] when no path in the set is a readable video + /// file, and anything [`Prober::probe`] raises that is about the prober + /// rather than about one file. + pub async fn select_feature( + &self, + paths: impl IntoIterator>, + expected_runtime: Option, + ) -> Result { + let mut candidates: Vec = Vec::new(); + for path in paths { + let path = path.into(); + match self.probe(path.clone()).await { + Ok(file) => candidates.push(file), + Err(error) if error.is_about_the_file() => { + tracing::debug!(path = %path.display(), %error, "not a video file, skipping"); + } + Err(error) => return Err(error), + } + } + + candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.size)); + + let matched = expected_runtime.and_then(|expected| { + candidates + .iter() + .position(|file| within_tolerance(file.duration, expected)) + }); + + if candidates.is_empty() { + return Err(Error::NoCandidates); + } + + let feature = candidates.remove(matched.unwrap_or(0)); + let runtime = match (expected_runtime, matched) { + (None, _) => RuntimeMatch::Unchecked, + (Some(_), Some(_)) => RuntimeMatch::Matched, + (Some(expected), None) => RuntimeMatch::Mismatched { + expected, + actual: feature.duration, + }, + }; + + Ok(FeatureSelection { + feature, + runtime, + others: candidates, + }) + } + + /// Spawn `ffprobe` and collect its stdout. + async fn run(&self, path: &Path) -> Result> { + let mut command = Command::new(&self.binary); + command + .args([ + "-v", + "error", + "-hide_banner", + "-print_format", + "json", + "-show_format", + "-show_streams", + "-i", + ]) + .arg(path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + let child = command.spawn().map_err(|source| Error::Spawn { + binary: self.binary.to_string_lossy().into_owned(), + source, + })?; + + let output = match tokio::time::timeout(self.timeout, child.wait_with_output()).await { + Ok(output) => output.map_err(|source| Error::Io { + path: path.to_path_buf(), + source, + })?, + Err(_) => { + return Err(Error::Timeout { + path: path.to_path_buf(), + }) + } + }; + + if output.status.success() { + return Ok(output.stdout); + } + + let stderr = String::from_utf8_lossy(&output.stderr) + .trim() + .chars() + .take(STDERR_LIMIT) + .collect(); + Err(Error::Rejected { + path: path.to_path_buf(), + status: output.status.code(), + stderr, + }) + } +} + +fn within_tolerance(actual: Duration, expected: Duration) -> bool { + actual.abs_diff(expected) <= expected.mul_f64(RUNTIME_TOLERANCE) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::within_tolerance; + + #[test] + fn an_extras_reel_is_not_the_feature() { + let feature = Duration::from_mins(120); + assert!(within_tolerance(Duration::from_mins(118), feature)); + assert!(!within_tolerance(Duration::from_mins(90), feature)); + } +} diff --git a/crates/arr-probe/src/model.rs b/crates/arr-probe/src/model.rs new file mode 100644 index 0000000..ad6097b --- /dev/null +++ b/crates/arr-probe/src/model.rs @@ -0,0 +1,260 @@ +//! What a probe returns. + +use std::{path::PathBuf, time::Duration}; + +use arr_core::{AudioTrack, DolbyVisionProfile, HdrFormat, ProbedMedia, Resolution, SubtitleTrack}; + +use crate::{ + error::{Error, Result}, + ffprobe::{Output, Stream}, + language, +}; + +/// The facts `ffprobe` reports about one file. +/// +/// [`ProbedFile::media`] is the part the policy engine consumes post-download +/// (DESIGN.md §5.6). The rest — size, duration, codec — is what naming, +/// feature selection and the UI need. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProbedFile { + /// The file that was probed. + pub path: PathBuf, + /// Size in bytes, as the container reports it. + pub size: u64, + /// Playing time. + pub duration: Duration, + /// `ffprobe`'s codec name for the video stream, e.g. `hevc` or `h264`. + pub video_codec: String, + /// Everything the policy engine looks at. + pub media: ProbedMedia, +} + +impl ProbedFile { + /// Build one from an `ffprobe` document. + /// + /// # Errors + /// + /// [`Error::NoVideoStream`] when the file carries no video, and + /// [`Error::NoDuration`] when neither the container nor any stream reports + /// a playing time. + pub(crate) fn from_output(path: PathBuf, output: &Output) -> Result { + if let Some(format) = output.format.name.as_deref() { + if let Some(name) = format.split(',').find(|name| is_still_or_text(name)) { + return Err(Error::NotVideo { + path, + format: name.to_owned(), + }); + } + } + + let video = output + .streams + .iter() + .find(|stream| stream.is_video()) + .ok_or_else(|| Error::NoVideoStream { path: path.clone() })?; + + let duration = duration(output).ok_or_else(|| Error::NoDuration { path: path.clone() })?; + + let size = output + .format + .size + .as_deref() + .and_then(|size| size.parse().ok()) + .unwrap_or_default(); + + let media = ProbedMedia { + resolution: resolution(video), + // A file cannot tell you where it came from. Source stays a claim + // of the release name (DESIGN.md §5.6). + source: None, + hdr: hdr(video), + audio_tracks: output + .streams + .iter() + .filter(|stream| stream.is_kind("audio")) + .map(audio_track) + .collect(), + subtitle_tracks: output + .streams + .iter() + .filter(|stream| stream.is_kind("subtitle")) + .map(subtitle_track) + .collect(), + }; + + Ok(Self { + path, + size, + duration, + video_codec: video.codec_name.clone().unwrap_or_default(), + media, + }) + } +} + +/// How a candidate's playing time compared to the runtime metadata claims. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RuntimeMatch { + /// Within tolerance of the expected runtime. + Matched, + /// No expected runtime was supplied, so nothing was checked. + Unchecked, + /// Every candidate was outside tolerance; the largest was taken anyway. + /// An extended cut looks like this, and so does an extras reel. + Mismatched { + /// What was expected, typically TMDB's runtime. + expected: Duration, + /// What the chosen file actually runs for. + actual: Duration, + }, +} + +/// Which file in a multi-file torrent is the feature. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FeatureSelection { + /// The chosen file. + pub feature: ProbedFile, + /// Whether the runtime sanity check agreed with the choice. + pub runtime: RuntimeMatch, + /// The other readable video files, largest first. Samples, extras and the + /// occasional second feature end up here. + pub others: Vec, +} + +/// Containers that are not video however hard `ffprobe` tries. +/// +/// The `tty` demuxer will read a `.nfo` as ANSI art and report a video stream +/// with a codec, a resolution and a duration, so "has a video stream" is not +/// on its own enough to call something importable. +fn is_still_or_text(format: &str) -> bool { + matches!(format, "tty" | "image2" | "gif" | "srt" | "ass" | "webvtt") + || format.ends_with("_pipe") +} + +fn audio_track(stream: &Stream) -> AudioTrack { + let title = stream.tags.get("title"); + let handler_name = stream.tags.get("handler_name"); + AudioTrack { + language: language::resolve(stream.tags.get("language"), title, handler_name), + title: title.map(str::to_owned), + handler_name: handler_name.map(str::to_owned), + } +} + +fn subtitle_track(stream: &Stream) -> SubtitleTrack { + SubtitleTrack { + language: language::resolve( + stream.tags.get("language"), + stream.tags.get("title"), + stream.tags.get("handler_name"), + ), + } +} + +/// Frame size to a resolution bucket. +/// +/// Width leads: a 2.39:1 scope transfer of a 2160p film is 3840x1600, and +/// bucketing that by height would call it 1080p. +fn resolution(video: &Stream) -> Resolution { + let width = video.width.unwrap_or_default(); + let height = video.height.unwrap_or_default(); + match () { + () if width >= 3000 || height >= 1700 => Resolution::R2160p, + () if width >= 1800 || height >= 900 => Resolution::R1080p, + () if width >= 1200 || height >= 620 => Resolution::R720p, + () => Resolution::Other(u16::try_from(height).unwrap_or(u16::MAX)), + } +} + +/// HDR format, Dolby Vision profile included. +/// +/// The profile is the whole point: DESIGN.md §5.3 accepts 8.1 and rejects 5 +/// and 7, and no release name can be trusted for it. +fn hdr(video: &Stream) -> HdrFormat { + if let Some(side_data) = video + .side_data_list + .iter() + .find(|side_data| side_data.dv_profile.is_some()) + { + if let Some(profile) = side_data.dv_profile { + return HdrFormat::DolbyVision(DolbyVisionProfile { + profile, + compatibility_id: side_data.dv_bl_signal_compatibility_id, + }); + } + } + + if video.side_data_list.iter().any(|side_data| { + side_data + .kind + .as_deref() + .is_some_and(|kind| kind.contains("2094")) + }) { + return HdrFormat::Hdr10Plus; + } + + match video.color_transfer.as_deref() { + Some("smpte2084") => HdrFormat::Hdr10, + Some("arib-std-b67") => HdrFormat::Hlg, + _ => HdrFormat::Sdr, + } +} + +/// Container duration first, then the longest stream, then Matroska's +/// per-stream `DURATION` tag, which is all a stream-copied remux carries. +fn duration(output: &Output) -> Option { + if let Some(duration) = output.format.duration.as_deref().and_then(seconds) { + return Some(duration); + } + + output + .streams + .iter() + .filter_map(|stream| { + stream + .duration + .as_deref() + .and_then(seconds) + .or_else(|| stream.tags.get("duration").and_then(timecode)) + }) + .max() +} + +/// `"1.023000"` — seconds as a float. +fn seconds(value: &str) -> Option { + let seconds: f64 = value.trim().parse().ok()?; + if seconds.is_finite() && seconds > 0.0 { + Duration::try_from_secs_f64(seconds).ok() + } else { + None + } +} + +/// `"00:00:01.023000000"` — Matroska's tag spelling. +fn timecode(value: &str) -> Option { + let mut parts = value.trim().split(':'); + let hours: u64 = parts.next()?.parse().ok()?; + let minutes: u64 = parts.next()?.parse().ok()?; + let seconds: f64 = parts.next()?.parse().ok()?; + if parts.next().is_some() || !seconds.is_finite() || seconds < 0.0 { + return None; + } + let whole = Duration::from_secs(hours * 3600 + minutes * 60); + Some(whole + Duration::try_from_secs_f64(seconds).ok()?) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::timecode; + + #[test] + fn matroska_timecodes_parse() { + assert_eq!( + timecode("00:01:30.500000000"), + Some(Duration::from_millis(90_500)) + ); + assert_eq!(timecode("nonsense"), None); + } +} diff --git a/crates/arr-probe/tests/fixtures/audio-with-cover.m4a b/crates/arr-probe/tests/fixtures/audio-with-cover.m4a new file mode 100644 index 0000000..8c03280 Binary files /dev/null and b/crates/arr-probe/tests/fixtures/audio-with-cover.m4a differ diff --git a/crates/arr-probe/tests/fixtures/dv-profile5.mp4 b/crates/arr-probe/tests/fixtures/dv-profile5.mp4 new file mode 100644 index 0000000..ed2cfa2 Binary files /dev/null and b/crates/arr-probe/tests/fixtures/dv-profile5.mp4 differ diff --git a/crates/arr-probe/tests/fixtures/dv-profile8-1.mp4 b/crates/arr-probe/tests/fixtures/dv-profile8-1.mp4 new file mode 100644 index 0000000..94fb8e3 Binary files /dev/null and b/crates/arr-probe/tests/fixtures/dv-profile8-1.mp4 differ diff --git a/crates/arr-probe/tests/fixtures/extras-360p.mkv b/crates/arr-probe/tests/fixtures/extras-360p.mkv new file mode 100644 index 0000000..ae0e9f6 Binary files /dev/null and b/crates/arr-probe/tests/fixtures/extras-360p.mkv differ diff --git a/crates/arr-probe/tests/fixtures/hdr10-2160p.mkv b/crates/arr-probe/tests/fixtures/hdr10-2160p.mkv new file mode 100644 index 0000000..275a8fd Binary files /dev/null and b/crates/arr-probe/tests/fixtures/hdr10-2160p.mkv differ diff --git a/crates/arr-probe/tests/fixtures/movie-1080p.mkv b/crates/arr-probe/tests/fixtures/movie-1080p.mkv new file mode 100644 index 0000000..95297db Binary files /dev/null and b/crates/arr-probe/tests/fixtures/movie-1080p.mkv differ diff --git a/crates/arr-probe/tests/fixtures/notes.nfo b/crates/arr-probe/tests/fixtures/notes.nfo new file mode 100644 index 0000000..6d94e12 --- /dev/null +++ b/crates/arr-probe/tests/fixtures/notes.nfo @@ -0,0 +1 @@ +Release notes. Not a video file. diff --git a/crates/arr-probe/tests/probe.rs b/crates/arr-probe/tests/probe.rs new file mode 100644 index 0000000..3d09732 --- /dev/null +++ b/crates/arr-probe/tests/probe.rs @@ -0,0 +1,243 @@ +//! Real `ffprobe` runs against the committed fixture clips (DESIGN.md §12). +//! +//! The clips are generated by `scripts/make-probe-fixtures.sh`. Every +//! assertion here is about output `ffprobe` actually produced, not a recorded +//! JSON document, because the shape of that output is exactly what breaks +//! between ffmpeg releases. + +// Same per-target quirk as elsewhere in the workspace: an integration test +// links the library's dependencies without using them directly. +use {serde as _, serde_json as _, thiserror as _, tracing as _}; + +use std::{path::PathBuf, time::Duration}; + +use arr_core::{DolbyVisionProfile, HdrFormat, Language, Resolution}; +use arr_probe::{Error, Prober, RuntimeMatch}; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(name) +} + +#[tokio::test] +async fn a_feature_reports_every_track_it_carries() { + let file = Prober::new() + .probe(fixture("movie-1080p.mkv")) + .await + .expect("the fixture is readable"); + + assert_eq!(file.media.resolution, Resolution::R1080p); + assert_eq!(file.video_codec, "h264"); + assert_eq!(file.media.hdr, HdrFormat::Sdr); + assert!(file.size > 0); + assert!(file.duration >= Duration::from_secs(3)); + + // A file never claims a source; that stays a claim of the release name. + assert_eq!(file.media.source, None); + + let languages: Vec<&Language> = file + .media + .audio_tracks + .iter() + .map(|track| &track.language) + .collect(); + assert_eq!( + languages, + vec![ + &Language::Other("en".to_owned()), + &Language::PortugueseBrazil, + &Language::PortuguesePortugal, + ] + ); + + let brazilian = &file.media.audio_tracks[1]; + assert_eq!(brazilian.title.as_deref(), Some("Português (Brasil)")); + assert_eq!( + brazilian.handler_name.as_deref(), + Some("Portuguese (Brazil)") + ); + + let subtitles: Vec<&Language> = file + .media + .subtitle_tracks + .iter() + .map(|track| &track.language) + .collect(); + assert_eq!( + subtitles, + vec![ + &Language::Other("en".to_owned()), + // `por` with nothing else to go on. Not guessed either way. + &Language::PortugueseUnverified, + ] + ); +} + +#[tokio::test] +async fn hdr10_comes_from_the_transfer_function() { + let file = Prober::new() + .probe(fixture("hdr10-2160p.mkv")) + .await + .expect("the fixture is readable"); + + assert_eq!(file.media.resolution, Resolution::R2160p); + assert_eq!(file.video_codec, "hevc"); + assert_eq!(file.media.hdr, HdrFormat::Hdr10); +} + +#[tokio::test] +async fn dolby_vision_profile_5_is_reported_as_a_number() { + let file = Prober::new() + .probe(fixture("dv-profile5.mp4")) + .await + .expect("the fixture is readable"); + + assert_eq!( + file.media.hdr, + HdrFormat::DolbyVision(DolbyVisionProfile { + profile: 5, + compatibility_id: Some(0), + }) + ); +} + +#[tokio::test] +async fn dolby_vision_profile_8_1_keeps_its_compatibility_id() { + let file = Prober::new() + .probe(fixture("dv-profile8-1.mp4")) + .await + .expect("the fixture is readable"); + + assert_eq!( + file.media.hdr, + HdrFormat::DolbyVision(DolbyVisionProfile { + profile: 8, + compatibility_id: Some(1), + }) + ); +} + +#[tokio::test] +async fn a_text_file_is_not_media() { + let error = Prober::new() + .probe(fixture("notes.nfo")) + .await + .expect_err("a text file is not media"); + + assert!(matches!(error, Error::NotVideo { .. }), "{error:?}"); +} + +/// Cover art is a video stream as far as ffprobe is concerned. +#[tokio::test] +async fn an_audio_file_with_artwork_is_not_video() { + let error = Prober::new() + .probe(fixture("audio-with-cover.m4a")) + .await + .expect_err("an attached picture is not a video stream"); + + assert!(matches!(error, Error::NoVideoStream { .. }), "{error:?}"); +} + +#[tokio::test] +async fn a_missing_file_is_an_error() { + let error = Prober::new() + .probe(fixture("does-not-exist.mkv")) + .await + .expect_err("nothing to probe"); + + assert!(matches!(error, Error::Rejected { .. }), "{error:?}"); +} + +#[tokio::test] +async fn a_missing_ffprobe_is_an_error() { + let error = Prober::new() + .with_binary("ffprobe-that-does-not-exist") + .probe(fixture("movie-1080p.mkv")) + .await + .expect_err("no binary to run"); + + assert!(matches!(error, Error::Spawn { .. }), "{error:?}"); +} + +/// The extras reel is the larger file on disk and the shorter one on the +/// clock. Size alone picks it; the runtime check does not. +#[tokio::test] +async fn runtime_beats_size_when_picking_the_feature() { + let selection = Prober::new() + .select_feature( + [ + fixture("extras-360p.mkv"), + fixture("movie-1080p.mkv"), + fixture("notes.nfo"), + ], + Some(Duration::from_secs(3)), + ) + .await + .expect("one of them is a video file"); + + assert_eq!(selection.feature.path, fixture("movie-1080p.mkv")); + assert_eq!(selection.runtime, RuntimeMatch::Matched); + assert_eq!(selection.others.len(), 1); + assert_eq!(selection.others[0].path, fixture("extras-360p.mkv")); + assert!(selection.others[0].size > selection.feature.size); +} + +#[tokio::test] +async fn without_an_expected_runtime_the_largest_file_wins() { + let selection = Prober::new() + .select_feature( + [fixture("movie-1080p.mkv"), fixture("extras-360p.mkv")], + None, + ) + .await + .expect("both are video files"); + + assert_eq!(selection.feature.path, fixture("extras-360p.mkv")); + assert_eq!(selection.runtime, RuntimeMatch::Unchecked); +} + +#[tokio::test] +async fn nothing_within_tolerance_still_returns_the_largest_file() { + let expected = Duration::from_hours(2); + let selection = Prober::new() + .select_feature( + [fixture("movie-1080p.mkv"), fixture("extras-360p.mkv")], + Some(expected), + ) + .await + .expect("both are video files"); + + assert_eq!(selection.feature.path, fixture("extras-360p.mkv")); + assert!( + matches!( + selection.runtime, + RuntimeMatch::Mismatched { expected: e, .. } if e == expected + ), + "{:?}", + selection.runtime + ); +} + +/// A prober that cannot run is not the same fact as a torrent without video, +/// and must not be reported as one. +#[tokio::test] +async fn a_broken_prober_fails_the_selection() { + let error = Prober::new() + .with_binary("ffprobe-that-does-not-exist") + .select_feature([fixture("movie-1080p.mkv")], None) + .await + .expect_err("nothing could be probed"); + + assert!(matches!(error, Error::Spawn { .. }), "{error:?}"); +} + +#[tokio::test] +async fn a_torrent_with_no_video_file_selects_nothing() { + let error = Prober::new() + .select_feature([fixture("notes.nfo")], None) + .await + .expect_err("no video among the candidates"); + + assert!(matches!(error, Error::NoCandidates), "{error:?}"); +} diff --git a/scripts/make-probe-fixtures.sh b/scripts/make-probe-fixtures.sh new file mode 100755 index 0000000..08af9e3 --- /dev/null +++ b/scripts/make-probe-fixtures.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Regenerate the ffprobe fixture clips in crates/arr-probe/tests/fixtures. +# +# The clips are committed (a few KB each, DESIGN.md §12) so the test suite +# never needs ffmpeg. This script exists to document how they were made and to +# rebuild them if the set ever needs to change. +# +# The two Dolby Vision clips cannot be encoded directly: x265 refuses to write +# a DV stream without an RPU file. Instead a real HEVC clip is produced and the +# `dvcC`/`dvvC` configuration box is spliced into its sample entry, which is +# exactly what ffprobe reads to report `dv_profile`. + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +out="$root/crates/arr-probe/tests/fixtures" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +mkdir -p "$out" + +# 1080p feature: three audio tracks covering all three §5.2 signals, plus two +# subtitle tracks. Three seconds, so the runtime sanity check has something to +# compare against. +printf '1\n00:00:00,000 --> 00:00:01,000\nhello\n\n' > "$work/en.srt" +printf '1\n00:00:00,000 --> 00:00:01,000\nolá\n\n' > "$work/pt.srt" + +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "color=c=black:s=1920x1080:r=24:d=3" \ + -f lavfi -i "sine=frequency=440:duration=3" \ + -f lavfi -i "sine=frequency=660:duration=3" \ + -f lavfi -i "sine=frequency=880:duration=3" \ + -i "$work/en.srt" -i "$work/pt.srt" \ + -map 0:v -map 1:a -map 2:a -map 3:a -map 4:s -map 5:s \ + -c:v libx264 -preset ultrafast -crf 51 -pix_fmt yuv420p \ + -c:a aac -b:a 8k -c:s srt \ + -metadata:s:a:0 language=eng -metadata:s:a:0 title="English" \ + -metadata:s:a:1 language=por -metadata:s:a:1 title="Português (Brasil)" \ + -metadata:s:a:1 handler_name="Portuguese (Brazil)" \ + -metadata:s:a:2 language=pt-PT \ + -metadata:s:s:0 language=eng -metadata:s:s:1 language=por \ + "$out/movie-1080p.mkv" + +# The extras reel trap: larger on disk than the feature (noise compresses +# badly) but a third of its runtime. +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "nullsrc=s=640x360:r=12:d=1,geq=random(1)*255:128:128" \ + -an -c:v libx264 -preset ultrafast -crf 51 -pix_fmt yuv420p \ + "$out/extras-360p.mkv" + +# HDR10: bt2020 primaries and the PQ transfer, which is all ffprobe reports. +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "color=c=black:s=3840x2160:r=24:d=1" -an \ + -vf "format=yuv420p10le,setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc" \ + -c:v libx265 -preset ultrafast -crf 51 \ + "$out/hdr10-2160p.mkv" + +# HEVC base layer for the two Dolby Vision clips. +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "color=c=black:s=1280x720:r=24:d=1" -an \ + -vf "format=yuv420p10le,setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc" \ + -c:v libx265 -preset ultrafast -crf 51 -tag:v hvc1 \ + "$work/base.mp4" + +cat > "$work/dovi.py" <<'PY' +"""Splice a DolbyVisionConfigurationBox into an MP4 HEVC sample entry.""" + +import struct +import sys + +CONTAINERS = {"moov", "trak", "mdia", "minf", "stbl", "stsd"} + + +def children(buf, start, end): + boxes = [] + off = start + while off + 8 <= end: + size = struct.unpack(">I", buf[off : off + 4])[0] + if size == 0: + size = end - off + boxes.append((off, size, buf[off + 4 : off + 8].decode("latin1"))) + off += size + return boxes + + +def first_child_offset(off, kind): + # stsd carries a version/flags word and an entry count before its children. + return off + (16 if kind == "stsd" else 8) + + +def find(buf, start, end, path): + for off, size, kind in children(buf, start, end): + if kind != path[0]: + continue + if len(path) == 1: + return off, size + return find(buf, first_child_offset(off, kind), off + size, path[1:]) + return None + + +def configuration(profile, compatibility_id, level=6): + bits = (profile & 0x7F) << 41 + bits |= (level & 0x3F) << 35 + bits |= 1 << 34 # rpu_present_flag + bits |= 1 << 32 # bl_present_flag + bits |= (compatibility_id & 0xF) << 28 + return bytes([1, 0]) + bits.to_bytes(6, "big") + bytes(16) + + +def grow_ancestors(buf, target, growth): + def walk(start, end): + for off, size, kind in children(buf, start, end): + if not off <= target < off + size: + continue + struct.pack_into(">I", buf, off, size + growth) + if kind in CONTAINERS: + walk(first_child_offset(off, kind), off + size) + return + + walk(0, len(buf)) + + +def main(source, destination, profile, compatibility_id): + buf = bytearray(open(source, "rb").read()) + entry = find(buf, 0, len(buf), ["moov", "trak", "mdia", "minf", "stbl", "stsd", "hvc1"]) + if entry is None: + sys.exit("no hvc1 sample entry") + off, size = entry + + payload = configuration(profile, compatibility_id) + kind = b"dvvC" if profile == 5 else b"dvcC" + box = struct.pack(">I", 8 + len(payload)) + kind + payload + + buf[off + 4 : off + 8] = b"dvh1" + buf[off + size : off + size] = box + grow_ancestors(buf, off, len(box)) + open(destination, "wb").write(bytes(buf)) + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])) +PY + +python3 "$work/dovi.py" "$work/base.mp4" "$out/dv-profile5.mp4" 5 0 +python3 "$work/dovi.py" "$work/base.mp4" "$out/dv-profile8-1.mp4" 8 1 + +# An audio file with embedded artwork. ffprobe reports the cover as a video +# stream, so this is what stops a soundtrack passing for a feature. +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "color=c=red:s=64x64:d=1" -frames:v 1 "$work/cover.jpg" +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "sine=frequency=440:duration=2" -i "$work/cover.jpg" \ + -map 0:a -map 1:v -c:a aac -b:a 8k -c:v mjpeg -disposition:v attached_pic \ + "$out/audio-with-cover.m4a" + +# Not media at all: the thing a multi-file torrent is full of. +printf 'Release notes. Not a video file.\n' > "$out/notes.nfo" + +ls -l "$out"