Import pipeline: probe, hardlink, rename, layout (#82)
This commit was merged in pull request #82.
This commit is contained in:
@@ -0,0 +1,991 @@
|
||||
//! The import pipeline: probe a completed download, judge it against the
|
||||
//! policy a second time with real evidence, and link it into the library. See
|
||||
//! DESIGN.md §5.7, §7.2, §7.3 and §7.4.
|
||||
//!
|
||||
//! The torrent's own files are never moved, renamed or deleted — the torrent
|
||||
//! and the library entry are separate lifecycles (§7.3). A hard-failed
|
||||
//! release is blacklisted and its grab marked failed, but the torrent keeps
|
||||
//! seeding until Transmission's own limits clear it.
|
||||
//!
|
||||
//! Everything here is idempotent from domain rows (§8): a grab in
|
||||
//! `downloaded` with no import recorded is the gap, and re-running any prefix
|
||||
//! of the pipeline after a crash converges — the hardlink call tolerates the
|
||||
//! destination already existing, and the `media_files` insert upserts on
|
||||
//! path.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::OsString;
|
||||
use std::io;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use arr_core::layout;
|
||||
use arr_core::policy::{evaluate, Candidate};
|
||||
use arr_core::{ProbedMedia, Rule, Source, Verdict};
|
||||
use arr_db::Db;
|
||||
use arr_dl::TransmissionClient;
|
||||
use arr_probe::Prober;
|
||||
|
||||
use crate::reconcile::{Action, ActionFuture, Outcome};
|
||||
|
||||
/// A failure during one import tick.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ImportError {
|
||||
#[error("database: {0}")]
|
||||
Database(#[from] sqlx::Error),
|
||||
#[error("policy: {0}")]
|
||||
Policy(#[from] arr_db::PolicyError),
|
||||
#[error("transmission: {0}")]
|
||||
Transmission(#[from] arr_dl::Error),
|
||||
#[error("probe: {0}")]
|
||||
Probe(#[from] arr_probe::Error),
|
||||
#[error("blocking task: {0}")]
|
||||
Join(#[from] tokio::task::JoinError),
|
||||
#[error("{action} {path}: {source}")]
|
||||
Io {
|
||||
action: &'static str,
|
||||
path: PathBuf,
|
||||
source: io::Error,
|
||||
},
|
||||
}
|
||||
|
||||
/// How the feature reached the library.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Placement {
|
||||
Linked,
|
||||
/// `link()` raised `EXDEV`; the file was copied instead (§7.2).
|
||||
Copied,
|
||||
/// The destination already existed — an earlier attempt placed it before
|
||||
/// the process died. Never a partial file: copies land via rename.
|
||||
AlreadyPlaced,
|
||||
}
|
||||
|
||||
/// What one probe attempt settled about one path.
|
||||
#[derive(Debug, Clone)]
|
||||
enum ProbeOutcome {
|
||||
Media(Box<arr_probe::ProbedFile>),
|
||||
/// A fact about the file — `.nfo`, artwork, corrupt — not the prober.
|
||||
NotMedia,
|
||||
}
|
||||
|
||||
/// Imports every downloaded grab: probe, second policy pass, hardlink into
|
||||
/// the §7.4 layout.
|
||||
#[derive(Debug)]
|
||||
pub struct ImportAction {
|
||||
transmission: TransmissionClient,
|
||||
prober: Prober,
|
||||
/// Probe results by path, kept across ticks. The reconcile lane cancels
|
||||
/// the whole action after its 25 s budget while one probe alone may take
|
||||
/// up to 60 s, so without this a large multi-file torrent would restart
|
||||
/// from the first file every tick and never finish. Shared with the
|
||||
/// detached probe tasks, which is what lets a probe outlive a cancelled
|
||||
/// tick and still deposit its result. Transient state, rebuilt by
|
||||
/// re-probing after a restart (§8); entries are dropped once their grab
|
||||
/// settles.
|
||||
probed: std::sync::Arc<tokio::sync::Mutex<HashMap<PathBuf, ProbeOutcome>>>,
|
||||
}
|
||||
|
||||
/// A movie grab Transmission finished downloading, not yet imported.
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingImport {
|
||||
grab_id: i64,
|
||||
infohash: String,
|
||||
movie_id: i64,
|
||||
tmdb_id: i64,
|
||||
title: String,
|
||||
year: Option<i64>,
|
||||
original_language: Option<String>,
|
||||
release_name: String,
|
||||
}
|
||||
|
||||
impl ImportAction {
|
||||
#[must_use]
|
||||
pub fn new(transmission: TransmissionClient, prober: Prober) -> Self {
|
||||
Self {
|
||||
transmission,
|
||||
prober,
|
||||
probed: std::sync::Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe every path, reusing results settled on earlier ticks, and
|
||||
/// return the readable video files.
|
||||
///
|
||||
/// Each probe runs as a detached task that writes the cache itself, so a
|
||||
/// tick cancelled by the lane's 25 s budget mid-probe loses nothing: the
|
||||
/// probe finishes in the background (its own 60 s limit still applies)
|
||||
/// and the next tick reads the deposited result. A tick that re-requests
|
||||
/// a path an orphaned probe is still on starts a second probe of the same
|
||||
/// file — bounded overlap, identical result, chosen over tracking
|
||||
/// in-flight probes.
|
||||
///
|
||||
/// Only facts about a file are cached; a prober failure — missing
|
||||
/// binary, timeout, unparseable output — is returned so the tick retries.
|
||||
async fn probe_all(
|
||||
&self,
|
||||
paths: &[PathBuf],
|
||||
) -> Result<Vec<arr_probe::ProbedFile>, ImportError> {
|
||||
let mut files = Vec::new();
|
||||
for path in paths {
|
||||
let cached = self.probed.lock().await.get(path).cloned();
|
||||
let outcome = if let Some(outcome) = cached {
|
||||
outcome
|
||||
} else {
|
||||
let prober = self.prober.clone();
|
||||
let cache = std::sync::Arc::clone(&self.probed);
|
||||
let target = path.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = match prober.probe(target.clone()).await {
|
||||
Ok(file) => ProbeOutcome::Media(Box::new(file)),
|
||||
Err(error) if error.is_about_the_file() => {
|
||||
tracing::debug!(path = %target.display(), %error, "not a video file, skipping");
|
||||
ProbeOutcome::NotMedia
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
cache.lock().await.insert(target, outcome.clone());
|
||||
Ok(outcome)
|
||||
})
|
||||
.await??
|
||||
};
|
||||
if let ProbeOutcome::Media(file) = outcome {
|
||||
files.push(*file);
|
||||
}
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Drop a settled grab's probe results — imported or blacklisted, they
|
||||
/// will not be needed again.
|
||||
async fn forget_probes(&self, paths: &[PathBuf]) {
|
||||
let mut probed = self.probed.lock().await;
|
||||
for path in paths {
|
||||
probed.remove(path);
|
||||
}
|
||||
}
|
||||
|
||||
async fn tick(&self, database: &Db) -> Result<Vec<Outcome>, ImportError> {
|
||||
let mut outcomes = Vec::new();
|
||||
for pending in pending_imports(database).await? {
|
||||
match self.import_one(database, &pending).await {
|
||||
Ok(Some(outcome)) => outcomes.push(outcome),
|
||||
Ok(None) => {}
|
||||
// One grab's failure must not cost the rest of the tick.
|
||||
Err(error) => tracing::error!(
|
||||
grab_id = pending.grab_id,
|
||||
movie_id = pending.movie_id,
|
||||
title = pending.title,
|
||||
%error,
|
||||
"import failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(outcomes)
|
||||
}
|
||||
|
||||
async fn import_one(
|
||||
&self,
|
||||
database: &Db,
|
||||
pending: &PendingImport,
|
||||
) -> Result<Option<Outcome>, ImportError> {
|
||||
let Some(loaded) = database.movie_policy(pending.movie_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// §5.2: no original language, nothing to judge audio against.
|
||||
let Some(original_language) = pending.original_language.as_deref() else {
|
||||
tracing::warn!(
|
||||
movie_id = pending.movie_id,
|
||||
title = pending.title,
|
||||
"no original language yet; not importing"
|
||||
);
|
||||
return Ok(None);
|
||||
};
|
||||
let original_language = arr_db::policy::language(original_language);
|
||||
|
||||
let Some(content) = self.transmission.torrent_content(&pending.infohash).await? else {
|
||||
// Gone from Transmission. Whether that is a failure or a manual
|
||||
// removal is issue #24's call; leave the grab alone.
|
||||
tracing::warn!(
|
||||
grab_id = pending.grab_id,
|
||||
infohash = pending.infohash,
|
||||
"downloaded grab has no torrent in Transmission; not importing"
|
||||
);
|
||||
return Ok(None);
|
||||
};
|
||||
// Torrent-declared names are untrusted input: an absolute or
|
||||
// `..`-carrying entry would escape the download root and get probed —
|
||||
// and possibly hardlinked — from anywhere on disk.
|
||||
let paths: Vec<PathBuf> = content
|
||||
.files
|
||||
.iter()
|
||||
.filter_map(|file| {
|
||||
let path = safe_join(&content.download_dir, &file.path);
|
||||
if path.is_none() {
|
||||
tracing::warn!(
|
||||
grab_id = pending.grab_id,
|
||||
path = %file.path.display(),
|
||||
"torrent file path escapes the download root; skipping"
|
||||
);
|
||||
}
|
||||
path
|
||||
})
|
||||
.collect();
|
||||
|
||||
// No expected runtime yet: the movies table carries no TMDB runtime,
|
||||
// so feature selection is by size alone (largest readable video).
|
||||
let mut candidates = self.probe_all(&paths).await?;
|
||||
candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.size));
|
||||
let Some(feature) = candidates.into_iter().next() else {
|
||||
// §5.7 "corrupt, wrong content": nothing in the torrent is a
|
||||
// readable video file.
|
||||
self.forget_probes(&paths).await;
|
||||
return self
|
||||
.hard_fail(database, pending, "no readable video file")
|
||||
.await
|
||||
.map(Some);
|
||||
};
|
||||
|
||||
// §5.6 second phase of truth: same policy, real evidence.
|
||||
let evaluation = evaluate(
|
||||
&loaded.policy,
|
||||
&loaded.overrides,
|
||||
&original_language,
|
||||
Candidate::PostDownload(&feature.media),
|
||||
Some(feature.size),
|
||||
);
|
||||
let waiver: Option<Rule> = match evaluation.verdict {
|
||||
Verdict::Rejected(rule) => {
|
||||
self.forget_probes(&paths).await;
|
||||
return self
|
||||
.hard_fail(database, pending, &rule.name())
|
||||
.await
|
||||
.map(Some);
|
||||
}
|
||||
Verdict::Waived(rule) => Some(rule),
|
||||
Verdict::Eligible => None,
|
||||
};
|
||||
|
||||
// The source tag is the one claim a file cannot verify (§5.6); every
|
||||
// other tag comes from the probe.
|
||||
let claimed_source = arr_parse::parse(&pending.release_name)
|
||||
.source
|
||||
.map(Source::from);
|
||||
let tags = layout::attribute_tags(&feature.media, claimed_source);
|
||||
let extension = feature.path.extension().and_then(|ext| ext.to_str());
|
||||
let folder = layout::movie_folder(&pending.title, pending.year, pending.tmdb_id);
|
||||
let file_name = layout::movie_file_name(
|
||||
&pending.title,
|
||||
pending.year,
|
||||
pending.tmdb_id,
|
||||
&tags,
|
||||
extension,
|
||||
);
|
||||
let destination = Path::new(&loaded.root_path).join(folder).join(file_name);
|
||||
|
||||
let source_path = feature.path.clone();
|
||||
let link_target = destination.clone();
|
||||
let placement =
|
||||
tokio::task::spawn_blocking(move || place(&source_path, &link_target)).await??;
|
||||
|
||||
record_import(database, pending, &feature, waiver.as_ref(), &destination).await?;
|
||||
self.forget_probes(&paths).await;
|
||||
let path_text = destination.to_string_lossy().into_owned();
|
||||
tracing::info!(
|
||||
grab_id = pending.grab_id,
|
||||
movie_id = pending.movie_id,
|
||||
title = pending.title,
|
||||
path = path_text,
|
||||
placement = ?placement,
|
||||
waived = waiver.is_some(),
|
||||
"imported"
|
||||
);
|
||||
Ok(Some(Outcome::new(
|
||||
format!("grab {} downloaded, not imported", pending.grab_id),
|
||||
format!("imported {path_text}"),
|
||||
)))
|
||||
}
|
||||
|
||||
/// §5.7 hard fail: blacklist the release, fail the grab, reopen the gap.
|
||||
/// The torrent is deliberately untouched (§7.3).
|
||||
async fn hard_fail(
|
||||
&self,
|
||||
database: &Db,
|
||||
pending: &PendingImport,
|
||||
reason: &str,
|
||||
) -> Result<Outcome, ImportError> {
|
||||
let normalised = arr_parse::normalise(&pending.release_name);
|
||||
sqlx::query!(
|
||||
"INSERT INTO blacklist (infohash, normalised_name, reason)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT (infohash) DO NOTHING",
|
||||
pending.infohash,
|
||||
normalised,
|
||||
reason
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"UPDATE grabs SET state = 'failed' WHERE id = ?",
|
||||
pending.grab_id
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"UPDATE movies
|
||||
SET state = 'missing',
|
||||
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
||||
WHERE id = ?",
|
||||
pending.movie_id
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
|
||||
tracing::warn!(
|
||||
grab_id = pending.grab_id,
|
||||
movie_id = pending.movie_id,
|
||||
title = pending.title,
|
||||
release = pending.release_name,
|
||||
reason,
|
||||
"hard fail post-probe; blacklisted, torrent left seeding"
|
||||
);
|
||||
Ok(Outcome::new(
|
||||
format!("grab {} hard-failed post-probe: {reason}", pending.grab_id),
|
||||
format!("blacklisted {}", pending.release_name),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl Action for ImportAction {
|
||||
fn name(&self) -> &'static str {
|
||||
"import"
|
||||
}
|
||||
|
||||
fn run<'a>(&'a self, database: &'a Db) -> ActionFuture<'a> {
|
||||
Box::pin(async move { self.tick(database).await.map_err(Into::into) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Settle a placed file into the rows: the `media_files` record (§4), the
|
||||
/// grab and the movie. The upsert on path is the crash seam — a re-run after
|
||||
/// a death between the link and here converges instead of erroring.
|
||||
async fn record_import(
|
||||
database: &Db,
|
||||
pending: &PendingImport,
|
||||
feature: &arr_probe::ProbedFile,
|
||||
waiver: Option<&Rule>,
|
||||
destination: &Path,
|
||||
) -> Result<(), ImportError> {
|
||||
let probed = probed_json(&feature.media).to_string();
|
||||
let waiver_json = waiver.map(|rule| serde_json::json!({ "rule": rule.name() }).to_string());
|
||||
let size = i64::try_from(feature.size).unwrap_or(i64::MAX);
|
||||
let path_text = destination.to_string_lossy().into_owned();
|
||||
sqlx::query!(
|
||||
"INSERT INTO media_files (owner_kind, owner_id, path, size, probed, waiver)
|
||||
VALUES ('movie', ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (path) DO UPDATE SET
|
||||
size = excluded.size,
|
||||
probed = excluded.probed,
|
||||
waiver = excluded.waiver,
|
||||
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')",
|
||||
pending.movie_id,
|
||||
path_text,
|
||||
size,
|
||||
probed,
|
||||
waiver_json
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"UPDATE grabs
|
||||
SET state = 'imported',
|
||||
imported_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
||||
WHERE id = ?",
|
||||
pending.grab_id
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"UPDATE movies
|
||||
SET state = 'imported',
|
||||
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
||||
WHERE id = ?",
|
||||
pending.movie_id
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The gap, straight out of the domain rows (§8): a movie grab Transmission
|
||||
/// finished that no import has settled.
|
||||
async fn pending_imports(database: &Db) -> Result<Vec<PendingImport>, ImportError> {
|
||||
let rows = sqlx::query!(
|
||||
r#"
|
||||
SELECT g.id AS "grab_id!: i64",
|
||||
g.infohash AS "infohash!: String",
|
||||
m.id AS "movie_id!: i64",
|
||||
m.tmdb_id AS "tmdb_id!: i64",
|
||||
m.title AS "title!: String",
|
||||
m.year,
|
||||
m.original_language,
|
||||
r.name AS "release_name!: String"
|
||||
FROM grabs g
|
||||
JOIN movies m ON m.id = g.target_id
|
||||
JOIN releases r ON r.id = g.release_id
|
||||
WHERE g.state = 'downloaded' AND g.target_kind = 'movie'
|
||||
ORDER BY g.id
|
||||
"#
|
||||
)
|
||||
.fetch_all(database.pool())
|
||||
.await?;
|
||||
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|row| PendingImport {
|
||||
grab_id: row.grab_id,
|
||||
infohash: row.infohash,
|
||||
movie_id: row.movie_id,
|
||||
tmdb_id: row.tmdb_id,
|
||||
title: row.title,
|
||||
year: row.year,
|
||||
original_language: row.original_language,
|
||||
release_name: row.release_name,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The `probed` column (§4, §5.6): what `ffprobe` found, in the spellings the
|
||||
/// policy columns use.
|
||||
fn probed_json(media: &ProbedMedia) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"resolution": media.resolution.to_string(),
|
||||
"source": media.source.map(|source| source.to_string()),
|
||||
"hdr": media.hdr.to_string(),
|
||||
"audio_tracks": media
|
||||
.audio_tracks
|
||||
.iter()
|
||||
.map(|track| serde_json::json!({
|
||||
"language": track.language.to_string(),
|
||||
"title": track.title,
|
||||
"handler_name": track.handler_name,
|
||||
}))
|
||||
.collect::<Vec<_>>(),
|
||||
"sub_tracks": media
|
||||
.subtitle_tracks
|
||||
.iter()
|
||||
.map(|track| serde_json::json!({ "language": track.language.to_string() }))
|
||||
.collect::<Vec<_>>(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Join a torrent-declared file path onto the download root, refusing
|
||||
/// anything that could land outside it: absolute paths, drive prefixes and
|
||||
/// `..` components. `None` means the entry is hostile or malformed.
|
||||
fn safe_join(root: &Path, declared: &Path) -> Option<PathBuf> {
|
||||
let mut clean = PathBuf::new();
|
||||
for component in declared.components() {
|
||||
match component {
|
||||
Component::Normal(part) => clean.push(part),
|
||||
Component::CurDir => {}
|
||||
Component::RootDir | Component::Prefix(_) | Component::ParentDir => return None,
|
||||
}
|
||||
}
|
||||
if clean.as_os_str().is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(root.join(clean))
|
||||
}
|
||||
|
||||
/// Hardlink `source` to `destination`, falling back to copy on `EXDEV` only
|
||||
/// (§7.2). No configuration flag.
|
||||
fn place(source: &Path, destination: &Path) -> Result<Placement, ImportError> {
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|error| ImportError::Io {
|
||||
action: "create library folder",
|
||||
path: parent.to_path_buf(),
|
||||
source: error,
|
||||
})?;
|
||||
}
|
||||
|
||||
match std::fs::hard_link(source, destination) {
|
||||
Ok(()) => Ok(Placement::Linked),
|
||||
// A completed earlier attempt: links and copies both land whole
|
||||
// (copies via rename), so an existing destination is a finished one.
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => Ok(Placement::AlreadyPlaced),
|
||||
Err(error) if error.kind() == io::ErrorKind::CrossesDevices => {
|
||||
copy_into_place(source, destination)
|
||||
}
|
||||
Err(error) => Err(ImportError::Io {
|
||||
action: "hardlink into",
|
||||
path: destination.to_path_buf(),
|
||||
source: error,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy through a dot-name in the destination folder, then rename, so the
|
||||
/// library never shows a partial file.
|
||||
fn copy_into_place(source: &Path, destination: &Path) -> Result<Placement, ImportError> {
|
||||
let mut temp_name = OsString::from(".");
|
||||
temp_name.push(destination.file_name().unwrap_or_default());
|
||||
temp_name.push(".partial");
|
||||
let temp = destination.with_file_name(temp_name);
|
||||
|
||||
let copied = std::fs::copy(source, &temp).and_then(|_| std::fs::rename(&temp, destination));
|
||||
if let Err(error) = copied {
|
||||
let _ = std::fs::remove_file(&temp);
|
||||
return Err(ImportError::Io {
|
||||
action: "copy into",
|
||||
path: destination.to_path_buf(),
|
||||
source: error,
|
||||
});
|
||||
}
|
||||
Ok(Placement::Copied)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde_json::json;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use super::*;
|
||||
|
||||
const INFOHASH: &str = "0123456789abcdef0123456789abcdef01234567";
|
||||
const RELEASE_NAME: &str = "Dune.Part.Two.2024.2160p.WEB-DL.DDP5.1.Atmos";
|
||||
|
||||
/// A 2160p HDR10 file with an English track — what the seeded main-movies
|
||||
/// policy accepts. The size is the container's claim, matching §5.5's
|
||||
/// band; the bytes on disk are tiny.
|
||||
const HDR10_PROBE: &str = r#"{
|
||||
"format": {"format_name": "matroska,webm", "duration": "9060.0", "size": "23622320128"},
|
||||
"streams": [
|
||||
{"codec_type": "video", "codec_name": "hevc", "width": 3840, "height": 1600,
|
||||
"color_transfer": "smpte2084"},
|
||||
{"codec_type": "audio", "codec_name": "eac3", "tags": {"language": "eng"}}
|
||||
]
|
||||
}"#;
|
||||
|
||||
/// The same file as a Dolby Vision Profile 5 stream — §5.3's hard reject.
|
||||
const DV5_PROBE: &str = r#"{
|
||||
"format": {"format_name": "matroska,webm", "duration": "9060.0", "size": "23622320128"},
|
||||
"streams": [
|
||||
{"codec_type": "video", "codec_name": "hevc", "width": 3840, "height": 1600,
|
||||
"color_transfer": "smpte2084",
|
||||
"side_data_list": [{"side_data_type": "DOVI configuration record", "dv_profile": 5}]},
|
||||
{"codec_type": "audio", "codec_name": "eac3", "tags": {"language": "eng"}}
|
||||
]
|
||||
}"#;
|
||||
|
||||
struct Harness {
|
||||
_dir: tempfile::TempDir,
|
||||
database: Db,
|
||||
downloads: PathBuf,
|
||||
library: PathBuf,
|
||||
action: ImportAction,
|
||||
_server: MockServer,
|
||||
}
|
||||
|
||||
/// An `ffprobe` stand-in: canned JSON for media, a `tty` document for the
|
||||
/// `.nfo`, so feature selection sees what the real binary would report.
|
||||
fn fake_ffprobe(directory: &Path, media_json: &str) -> PathBuf {
|
||||
let path = directory.join("ffprobe");
|
||||
let script = format!(
|
||||
"#!/bin/sh\nfor arg; do last=\"$arg\"; done\ncase \"$last\" in\n *.nfo) printf '%s' '{{\"format\":{{\"format_name\":\"tty\"}}}}' ;;\n *) cat <<'PROBE_EOF'\n{media_json}\nPROBE_EOF\n;;\nesac\n"
|
||||
);
|
||||
std::fs::write(&path, script).unwrap();
|
||||
let mut permissions = std::fs::metadata(&path).unwrap().permissions();
|
||||
permissions.set_mode(0o755);
|
||||
std::fs::set_permissions(&path, permissions).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
async fn harness(media_json: &str) -> Harness {
|
||||
harness_with(
|
||||
media_json,
|
||||
json!([
|
||||
{"name": "Dune/Dune.mkv", "length": 13, "bytesCompleted": 13},
|
||||
{"name": "Dune/Dune.nfo", "length": 10, "bytesCompleted": 10}
|
||||
]),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn harness_with(media_json: &str, files: serde_json::Value) -> Harness {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let downloads = dir.path().join("downloads");
|
||||
let library = dir.path().join("library");
|
||||
std::fs::create_dir_all(downloads.join("Dune")).unwrap();
|
||||
std::fs::create_dir_all(&library).unwrap();
|
||||
std::fs::write(downloads.join("Dune/Dune.mkv"), b"feature bytes").unwrap();
|
||||
std::fs::write(downloads.join("Dune/Dune.nfo"), b"not a film").unwrap();
|
||||
|
||||
let database = Db::connect(dir.path().join("arr.db")).await.unwrap();
|
||||
database.migrate().await.unwrap();
|
||||
let library_text = library.to_string_lossy().into_owned();
|
||||
sqlx::query("UPDATE roots SET path = ? WHERE kind = 'movie' AND audience = 'main'")
|
||||
.bind(&library_text)
|
||||
.execute(database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO movies (tmdb_id, title, year, original_language, root_id, state)
|
||||
SELECT 693134, 'Dune: Part Two', 2024, 'en', id, 'grabbed'
|
||||
FROM roots WHERE kind = 'movie' AND audience = 'main'",
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let release_id = sqlx::query(
|
||||
"INSERT INTO releases (indexer_id, guid, name, size, download_url, parsed, verdict)
|
||||
VALUES (7, 'good', ?, 23622320128, 'magnet:x', '{}', 'eligible')",
|
||||
)
|
||||
.bind(RELEASE_NAME)
|
||||
.execute(database.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.last_insert_rowid();
|
||||
sqlx::query(
|
||||
"INSERT INTO grabs (release_id, target_kind, target_id, infohash, state)
|
||||
VALUES (?, 'movie', 1, ?, 'downloaded')",
|
||||
)
|
||||
.bind(release_id)
|
||||
.bind(INFOHASH)
|
||||
.execute(database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"result": "success",
|
||||
"arguments": {"torrents": [{
|
||||
"hashString": INFOHASH,
|
||||
"downloadDir": downloads.to_string_lossy(),
|
||||
"files": files
|
||||
}]}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let prober = Prober::new().with_binary(fake_ffprobe(dir.path(), media_json));
|
||||
let action = ImportAction::new(TransmissionClient::new(&server.uri()).unwrap(), prober);
|
||||
|
||||
Harness {
|
||||
_dir: dir,
|
||||
database,
|
||||
downloads,
|
||||
library,
|
||||
action,
|
||||
_server: server,
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_library_file(library: &Path) -> PathBuf {
|
||||
library
|
||||
.join("Dune Part Two (2024) [tmdbid-693134]")
|
||||
.join("Dune Part Two (2024) [tmdbid-693134] - [2160p][WEB-DL][HDR10].mkv")
|
||||
}
|
||||
|
||||
/// The issue's acceptance case: the exact §7.4 path exists, and the
|
||||
/// torrent's own file still exists with a link count of two.
|
||||
#[tokio::test]
|
||||
async fn the_feature_lands_on_the_design_layout_and_keeps_seeding() {
|
||||
let h = harness(HDR10_PROBE).await;
|
||||
|
||||
let outcomes = h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
let library_file = expected_library_file(&h.library);
|
||||
assert!(library_file.is_file(), "missing {}", library_file.display());
|
||||
|
||||
let seeding_file = h.downloads.join("Dune/Dune.mkv");
|
||||
let metadata = std::fs::metadata(&seeding_file).unwrap();
|
||||
assert_eq!(metadata.nlink(), 2, "§7.2: hardlinked, not moved or copied");
|
||||
|
||||
let (path, probed, waiver): (String, String, Option<String>) =
|
||||
sqlx::query_as("SELECT path, probed, waiver FROM media_files WHERE owner_kind = 'movie' AND owner_id = 1")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(path, library_file.to_string_lossy());
|
||||
assert!(probed.contains("\"2160p\""), "{probed}");
|
||||
assert!(probed.contains("HDR10"), "{probed}");
|
||||
assert_eq!(waiver, None);
|
||||
|
||||
let (grab_state, imported_at): (String, Option<String>) =
|
||||
sqlx::query_as("SELECT state, imported_at FROM grabs WHERE infohash = ?")
|
||||
.bind(INFOHASH)
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(grab_state, "imported");
|
||||
assert!(imported_at.is_some());
|
||||
let movie_state: String = sqlx::query_scalar("SELECT state FROM movies WHERE id = 1")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(movie_state, "imported");
|
||||
}
|
||||
|
||||
/// §5.3 through §5.7: Profile 5 is a hard fail — blacklisted, grab
|
||||
/// failed, gap reopened, and the torrent's files untouched.
|
||||
#[tokio::test]
|
||||
async fn a_dolby_vision_profile_5_file_hard_fails() {
|
||||
let h = harness(DV5_PROBE).await;
|
||||
|
||||
let outcomes = h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
assert!(
|
||||
std::fs::read_dir(&h.library).unwrap().next().is_none(),
|
||||
"nothing may reach the library"
|
||||
);
|
||||
assert!(h.downloads.join("Dune/Dune.mkv").is_file(), "§7.3");
|
||||
|
||||
let (infohash, normalised, reason): (String, String, String) =
|
||||
sqlx::query_as("SELECT infohash, normalised_name, reason FROM blacklist")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(infohash, INFOHASH);
|
||||
assert_eq!(normalised, arr_parse::normalise(RELEASE_NAME));
|
||||
assert_eq!(reason, "dolby_vision_profile");
|
||||
|
||||
let grab_state: String = sqlx::query_scalar("SELECT state FROM grabs")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(grab_state, "failed");
|
||||
let movie_state: String = sqlx::query_scalar("SELECT state FROM movies WHERE id = 1")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
movie_state, "missing",
|
||||
"the gap reopens for the next candidate"
|
||||
);
|
||||
}
|
||||
|
||||
/// §8: killed between the hardlink and the bookkeeping, a restart
|
||||
/// converges instead of failing on the existing destination.
|
||||
#[tokio::test]
|
||||
async fn a_restart_after_the_link_converges() {
|
||||
let h = harness(HDR10_PROBE).await;
|
||||
h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
// The crash: the file is placed, the database never heard.
|
||||
sqlx::query("DELETE FROM media_files")
|
||||
.execute(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE grabs SET state = 'downloaded', imported_at = NULL")
|
||||
.execute(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE movies SET state = 'grabbed'")
|
||||
.execute(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let outcomes = h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
let files: i64 = sqlx::query_scalar("SELECT count(*) FROM media_files")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(files, 1);
|
||||
let metadata = std::fs::metadata(h.downloads.join("Dune/Dune.mkv")).unwrap();
|
||||
assert_eq!(metadata.nlink(), 2, "no second link, no copy");
|
||||
}
|
||||
|
||||
/// A settled tick is idle — an imported grab is not a gap.
|
||||
#[tokio::test]
|
||||
async fn a_second_tick_imports_nothing_new() {
|
||||
let h = harness(HDR10_PROBE).await;
|
||||
h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
let outcomes = h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
assert!(outcomes.is_empty());
|
||||
}
|
||||
|
||||
/// Torrent-declared names are untrusted: absolute and `..`-carrying
|
||||
/// entries are skipped, and the import proceeds from what remains.
|
||||
#[tokio::test]
|
||||
async fn hostile_torrent_paths_never_leave_the_download_root() {
|
||||
let h = harness_with(
|
||||
HDR10_PROBE,
|
||||
json!([
|
||||
{"name": "../outside.mkv", "length": 13, "bytesCompleted": 13},
|
||||
{"name": "/tmp/absolute.mkv", "length": 13, "bytesCompleted": 13},
|
||||
{"name": "Dune/Dune.mkv", "length": 13, "bytesCompleted": 13}
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcomes = h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
assert!(expected_library_file(&h.library).is_file());
|
||||
let files: i64 = sqlx::query_scalar("SELECT count(*) FROM media_files")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(files, 1, "only the safe path is imported");
|
||||
}
|
||||
|
||||
/// A torrent whose every entry escapes the root has nothing importable:
|
||||
/// hard fail, not an escape.
|
||||
#[tokio::test]
|
||||
async fn a_torrent_of_only_hostile_paths_hard_fails() {
|
||||
let h = harness_with(
|
||||
HDR10_PROBE,
|
||||
json!([{"name": "../../etc/passwd", "length": 13, "bytesCompleted": 13}]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcomes = h.action.tick(&h.database).await.unwrap();
|
||||
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
let reason: String = sqlx::query_scalar("SELECT reason FROM blacklist")
|
||||
.fetch_one(h.database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(reason, "no readable video file");
|
||||
assert!(std::fs::read_dir(&h.library).unwrap().next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_join_refuses_escapes_and_keeps_normal_paths() {
|
||||
let root = Path::new("/downloads");
|
||||
assert_eq!(
|
||||
safe_join(root, Path::new("Dune/./Dune.mkv")),
|
||||
Some(PathBuf::from("/downloads/Dune/Dune.mkv"))
|
||||
);
|
||||
assert_eq!(safe_join(root, Path::new("../outside.mkv")), None);
|
||||
assert_eq!(safe_join(root, Path::new("Dune/../../outside.mkv")), None);
|
||||
assert_eq!(safe_join(root, Path::new("/etc/passwd")), None);
|
||||
assert_eq!(safe_join(root, Path::new("")), None);
|
||||
}
|
||||
|
||||
/// The reconcile lane cancels the action after 25 s while one probe may
|
||||
/// take 60 s, so results settled on one tick must survive to the next —
|
||||
/// otherwise a large torrent restarts from file one forever.
|
||||
#[tokio::test]
|
||||
async fn probe_results_are_reused_across_calls() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let media = dir.path().join("film.mkv");
|
||||
std::fs::write(&media, b"bytes").unwrap();
|
||||
let counter = dir.path().join("count");
|
||||
let script = dir.path().join("ffprobe");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/bin/sh\necho x >> {}\ncat <<'PROBE_EOF'\n{HDR10_PROBE}\nPROBE_EOF\n",
|
||||
counter.display()
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let mut permissions = std::fs::metadata(&script).unwrap().permissions();
|
||||
permissions.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, permissions).unwrap();
|
||||
|
||||
let action = ImportAction::new(
|
||||
TransmissionClient::new("http://127.0.0.1:1").unwrap(),
|
||||
Prober::new().with_binary(&script),
|
||||
);
|
||||
let paths = vec![media];
|
||||
|
||||
action.probe_all(&paths).await.unwrap();
|
||||
action.probe_all(&paths).await.unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(&counter).unwrap().lines().count(),
|
||||
1,
|
||||
"the second call reuses the first call's result"
|
||||
);
|
||||
|
||||
action.forget_probes(&paths).await;
|
||||
action.probe_all(&paths).await.unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(&counter).unwrap().lines().count(),
|
||||
2,
|
||||
"a settled grab's entries are dropped"
|
||||
);
|
||||
}
|
||||
|
||||
/// The probe outlives a cancelled tick: the detached task deposits its
|
||||
/// result after the caller's future is dropped, and the next tick reads
|
||||
/// it instead of restarting the same probe forever.
|
||||
#[tokio::test]
|
||||
async fn a_cancelled_probe_still_deposits_its_result() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let media = dir.path().join("film.mkv");
|
||||
std::fs::write(&media, b"bytes").unwrap();
|
||||
let counter = dir.path().join("count");
|
||||
let script = dir.path().join("ffprobe");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/bin/sh\nsleep 0.3\necho x >> {}\ncat <<'PROBE_EOF'\n{HDR10_PROBE}\nPROBE_EOF\n",
|
||||
counter.display()
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let mut permissions = std::fs::metadata(&script).unwrap().permissions();
|
||||
permissions.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, permissions).unwrap();
|
||||
let action = ImportAction::new(
|
||||
TransmissionClient::new("http://127.0.0.1:1").unwrap(),
|
||||
Prober::new().with_binary(&script),
|
||||
);
|
||||
let paths = vec![media];
|
||||
|
||||
// The lane's budget, in miniature: the tick is cancelled mid-probe.
|
||||
let cancelled = tokio::time::timeout(
|
||||
std::time::Duration::from_millis(50),
|
||||
action.probe_all(&paths),
|
||||
)
|
||||
.await;
|
||||
assert!(cancelled.is_err());
|
||||
|
||||
// The detached probe finishes on its own and deposits the result.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(600)).await;
|
||||
let files = action.probe_all(&paths).await.unwrap();
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(&counter).unwrap().lines().count(),
|
||||
1,
|
||||
"the next tick reused the deposited result instead of re-probing"
|
||||
);
|
||||
}
|
||||
|
||||
/// The `EXDEV` fallback path lands whole files via rename (§7.2).
|
||||
#[test]
|
||||
fn the_copy_fallback_lands_a_whole_file_and_cleans_up() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let source = dir.path().join("source.mkv");
|
||||
std::fs::write(&source, b"feature bytes").unwrap();
|
||||
let destination = dir.path().join("library").join("feature.mkv");
|
||||
std::fs::create_dir_all(destination.parent().unwrap()).unwrap();
|
||||
|
||||
let placement = copy_into_place(&source, &destination).unwrap();
|
||||
|
||||
assert_eq!(placement, Placement::Copied);
|
||||
assert_eq!(std::fs::read(&destination).unwrap(), b"feature bytes");
|
||||
assert!(
|
||||
std::fs::read_dir(destination.parent().unwrap())
|
||||
.unwrap()
|
||||
.all(|entry| !entry
|
||||
.unwrap()
|
||||
.file_name()
|
||||
.to_string_lossy()
|
||||
.contains("partial")),
|
||||
"no partial file left behind"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user