feat(db): SQLite schema and movie migrations (#48)
This commit was merged in pull request #48.
This commit is contained in:
@@ -7,6 +7,11 @@ repository.workspace = true
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
sqlx = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
-- Movie side of the domain model in DESIGN.md §4. Series, seasons and
|
||||
-- episodes are issue #34 and are deliberately absent here.
|
||||
--
|
||||
-- Every table is STRICT: SQLite's default type affinity would happily store
|
||||
-- a string in an INTEGER column, and this schema is the last place that
|
||||
-- should be forgiving.
|
||||
|
||||
CREATE TABLE policies (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
-- §5.2. Which audio track a release must carry, expressed against the
|
||||
-- title's original language rather than a fixed list.
|
||||
required_audio TEXT NOT NULL CHECK (json_valid(required_audio)),
|
||||
-- §5.2. Languages that never satisfy required_audio unless they are the
|
||||
-- title's own original language. Subtitles are not filtered.
|
||||
dub_blacklist TEXT NOT NULL CHECK (json_valid(dub_blacklist)),
|
||||
-- §5.3. Dolby Vision profiles, knowable only from ffprobe.
|
||||
hdr_rules TEXT NOT NULL CHECK (json_valid(hdr_rules)),
|
||||
-- §5.5. Per resolution: floor, target and the penalty above target.
|
||||
size_bands TEXT NOT NULL CHECK (json_valid(size_bands)),
|
||||
resolution_pref TEXT NOT NULL CHECK (json_valid(resolution_pref)),
|
||||
-- §5.5. Small tiebreaker, not the dominant scoring term.
|
||||
source_weights TEXT NOT NULL CHECK (json_valid(source_weights)),
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||
) STRICT;
|
||||
|
||||
-- §5.1. One policy per root, two roots per media kind.
|
||||
CREATE TABLE roots (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('movie', 'tv')),
|
||||
audience TEXT NOT NULL CHECK (audience IN ('main', 'kids')),
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
policy_id INTEGER NOT NULL REFERENCES policies (id),
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
UNIQUE (kind, audience)
|
||||
) STRICT;
|
||||
|
||||
-- §4.1. `wanted` is the only intent. `state` is a cache of what reconcile
|
||||
-- last observed, and arr-core (#7) owns the canonical enum this mirrors.
|
||||
CREATE TABLE movies (
|
||||
id INTEGER PRIMARY KEY,
|
||||
tmdb_id INTEGER NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
year INTEGER,
|
||||
-- §5.2. BCP-47 from TMDB. Nullable until the first metadata refresh.
|
||||
original_language TEXT,
|
||||
root_id INTEGER NOT NULL REFERENCES roots (id),
|
||||
wanted INTEGER NOT NULL DEFAULT 1 CHECK (wanted IN (0, 1)),
|
||||
-- §5.1. Per-title relaxations and tightenings of the root policy.
|
||||
overrides TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(overrides)),
|
||||
state TEXT NOT NULL DEFAULT 'missing'
|
||||
CHECK (state IN ('missing', 'grabbed', 'imported')),
|
||||
-- §6.3. Stops targeted search while leaving RSS matching on.
|
||||
blocked INTEGER NOT NULL DEFAULT 0 CHECK (blocked IN (0, 1)),
|
||||
search_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_searched_at TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||
) STRICT;
|
||||
|
||||
-- §8. The reconcile loop's work list is "wanted, not blocked, least recently
|
||||
-- searched first", so the partial index carries the sort column.
|
||||
CREATE INDEX movies_pending_search
|
||||
ON movies (last_searched_at)
|
||||
WHERE wanted = 1 AND blocked = 0;
|
||||
|
||||
CREATE INDEX movies_state ON movies (state);
|
||||
CREATE INDEX movies_root ON movies (root_id);
|
||||
|
||||
-- §4. Polymorphic owner so episodes (#34) reuse the table unchanged.
|
||||
CREATE TABLE media_files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
owner_kind TEXT NOT NULL CHECK (owner_kind IN ('movie', 'episode')),
|
||||
owner_id INTEGER NOT NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
size INTEGER NOT NULL,
|
||||
-- §5.6. What ffprobe found: resolution, source, hdr, audio and sub tracks.
|
||||
probed TEXT CHECK (probed IS NULL OR json_valid(probed)),
|
||||
-- §5.7. Which rule was relaxed to allow this import, if any.
|
||||
waiver TEXT CHECK (waiver IS NULL OR json_valid(waiver)),
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX media_files_owner ON media_files (owner_kind, owner_id);
|
||||
|
||||
-- §8. A file with no probe result yet is a pending import.
|
||||
CREATE INDEX media_files_unprobed ON media_files (id) WHERE probed IS NULL;
|
||||
|
||||
CREATE TABLE releases (
|
||||
id INTEGER PRIMARY KEY,
|
||||
-- Prowlarr's indexer id. Not a foreign key: indexers live in Prowlarr.
|
||||
indexer_id INTEGER NOT NULL,
|
||||
guid TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
seeders INTEGER,
|
||||
publish_date TEXT,
|
||||
download_url TEXT NOT NULL,
|
||||
-- §5.6. What the release name claims, before anything is downloaded.
|
||||
parsed TEXT NOT NULL CHECK (json_valid(parsed)),
|
||||
score REAL,
|
||||
-- §9.3. Three buckets. `rejected_rule` names the rule that killed it so
|
||||
-- an over-strict filter is visible without reading release names.
|
||||
verdict TEXT CHECK (verdict IN ('eligible', 'waived', 'rejected')),
|
||||
rejected_rule TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
UNIQUE (indexer_id, guid),
|
||||
CHECK ((verdict = 'rejected') = (rejected_rule IS NOT NULL))
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX releases_verdict ON releases (verdict, score);
|
||||
|
||||
-- §7.3. The torrent and the library entry are separate state machines; this
|
||||
-- row is the torrent's. Download progress stays in memory.
|
||||
CREATE TABLE grabs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
release_id INTEGER NOT NULL REFERENCES releases (id),
|
||||
target_kind TEXT NOT NULL CHECK (target_kind IN ('movie', 'episode')),
|
||||
target_id INTEGER NOT NULL,
|
||||
infohash TEXT NOT NULL UNIQUE,
|
||||
state TEXT NOT NULL DEFAULT 'sent'
|
||||
CHECK (state IN ('sent', 'downloaded', 'imported', 'failed')),
|
||||
grabbed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
||||
imported_at TEXT
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX grabs_state ON grabs (state);
|
||||
CREATE INDEX grabs_target ON grabs (target_kind, target_id);
|
||||
|
||||
-- §6.3. Keyed on infohash and on normalised name, because the same release
|
||||
-- reappears under a different infohash.
|
||||
CREATE TABLE blacklist (
|
||||
id INTEGER PRIMARY KEY,
|
||||
infohash TEXT UNIQUE,
|
||||
normalised_name TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX blacklist_name ON blacklist (normalised_name);
|
||||
|
||||
-- §4.3. A tag on titles, driving filtering and notification routing. Never a path.
|
||||
CREATE TABLE owners (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
ntfy_topic TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE title_owners (
|
||||
title_kind TEXT NOT NULL CHECK (title_kind IN ('movie', 'series')),
|
||||
title_id INTEGER NOT NULL,
|
||||
owner_id INTEGER NOT NULL REFERENCES owners (id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (title_kind, title_id, owner_id)
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX title_owners_owner ON title_owners (owner_id);
|
||||
@@ -0,0 +1,45 @@
|
||||
-- The two movie roots and their policies, DESIGN.md §5.1. Paths follow §7.4:
|
||||
-- media kind first, audience second.
|
||||
--
|
||||
-- Policy lives in the database rather than config (§10) because these numbers
|
||||
-- get tuned by hand. The size bands are placeholders pending the remux
|
||||
-- playback test in §14; the model is the decision, not the constants.
|
||||
|
||||
INSERT INTO policies (
|
||||
name, required_audio, dub_blacklist, hdr_rules,
|
||||
size_bands, resolution_pref, source_weights
|
||||
) VALUES (
|
||||
'Movies — main',
|
||||
-- §5.2. Require a track in the title's own original language. Anything
|
||||
-- else may ride along.
|
||||
json('{"require":"original_language","allow_english_fallback":false}'),
|
||||
json('["pt-BR"]'),
|
||||
-- §5.3. Profile 5 has no HDR10 base layer and Profile 7 depends on the
|
||||
-- player. Profile 8.1 plays as HDR10 and is fine.
|
||||
json('{"dv_profile_allow":["8.1"],"dv_profile_reject":["5","7"],"allow_hdr10":true,"allow_hdr10plus":true,"allow_sdr":true}'),
|
||||
-- Floors matter: unbounded "smaller is better" picks a 3 GB 4K encode
|
||||
-- that looks like mud.
|
||||
json('{"2160p":{"floor_gb":8,"target_gb":22,"penalty_per_gb_over":0.5},"1080p":{"floor_gb":3,"target_gb":8,"penalty_per_gb_over":0.5}}'),
|
||||
json('["2160p","1080p"]'),
|
||||
json('{"Remux":4,"BluRay":3,"WEB-DL":2,"WEBRip":1,"HDTV":0}')
|
||||
), (
|
||||
'Movies — kids',
|
||||
-- §5.2. pt-PT, or the original language when that is already Portuguese.
|
||||
-- pt-BR never satisfies this.
|
||||
json('{"require":"any_of","langs":["pt-PT"],"or_original_when_portuguese":true,"allow_english_fallback":false}'),
|
||||
json('["pt-BR"]'),
|
||||
json('{"dv_profile_allow":["8.1"],"dv_profile_reject":["5","7"],"allow_hdr10":true,"allow_hdr10plus":true,"allow_sdr":true}'),
|
||||
json('{"2160p":{"floor_gb":8,"target_gb":22,"penalty_per_gb_over":0.5},"1080p":{"floor_gb":3,"target_gb":8,"penalty_per_gb_over":0.5}}'),
|
||||
json('["2160p","1080p"]'),
|
||||
-- §5.2. European Portuguese dubs live in streaming WEB-DLs and
|
||||
-- essentially never in BluRay encodes, so this root biases hard that way.
|
||||
json('{"WEB-DL":4,"WEBRip":2,"BluRay":1,"Remux":1,"HDTV":0}')
|
||||
);
|
||||
|
||||
INSERT INTO roots (kind, audience, path, policy_id)
|
||||
SELECT 'movie', 'main', '/mnt/media/movies/main', id
|
||||
FROM policies WHERE name = 'Movies — main';
|
||||
|
||||
INSERT INTO roots (kind, audience, path, policy_id)
|
||||
SELECT 'movie', 'kids', '/mnt/media/movies/kids', id
|
||||
FROM policies WHERE name = 'Movies — kids';
|
||||
+221
-1
@@ -1 +1,221 @@
|
||||
//! arr-db — see DESIGN.md.
|
||||
//! arr-db — SQLite persistence and migrations. See DESIGN.md §10.
|
||||
//!
|
||||
//! A few thousand rows and a single writer, so the pool exists for readers
|
||||
//! and the schema does the enforcing: foreign keys on, WAL, `STRICT` tables.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous};
|
||||
use sqlx::{migrate::MigrateError, SqlitePool};
|
||||
|
||||
/// The migrations embedded in the binary, so a deploy is one file.
|
||||
pub static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations");
|
||||
|
||||
/// How long a writer waits for the write lock before giving up.
|
||||
const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// A connection pool with the pragmas this schema assumes already applied.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Db {
|
||||
pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Open (creating if absent) the database at `path`.
|
||||
///
|
||||
/// Every connection gets `foreign_keys` on — SQLite defaults it *off* per
|
||||
/// connection, so setting it anywhere but here is a way to forget it.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// If the file cannot be created or opened.
|
||||
pub async fn connect(path: impl AsRef<Path>) -> Result<Self, sqlx::Error> {
|
||||
let options = SqliteConnectOptions::new()
|
||||
.filename(path)
|
||||
.create_if_missing(true)
|
||||
.foreign_keys(true)
|
||||
.journal_mode(SqliteJournalMode::Wal)
|
||||
// WAL makes NORMAL durable against process crashes, which is the
|
||||
// failure this actually has. FULL costs an fsync per commit.
|
||||
.synchronous(SqliteSynchronous::Normal)
|
||||
.busy_timeout(BUSY_TIMEOUT);
|
||||
|
||||
let pool = SqlitePoolOptions::new().connect_with(options).await?;
|
||||
|
||||
Ok(Self { pool })
|
||||
}
|
||||
|
||||
/// Apply every migration that has not run yet.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// If a migration fails, or one already applied no longer matches its
|
||||
/// recorded checksum.
|
||||
pub async fn migrate(&self) -> Result<(), MigrateError> {
|
||||
MIGRATOR.run(&self.pool).await
|
||||
}
|
||||
|
||||
/// The underlying pool, for crates that own their own queries.
|
||||
#[must_use]
|
||||
pub fn pool(&self) -> &SqlitePool {
|
||||
&self.pool
|
||||
}
|
||||
|
||||
/// Every configured root with the policy attached to it (§5.1).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// If the query fails.
|
||||
pub async fn list_roots(&self) -> Result<Vec<Root>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
Root,
|
||||
r#"
|
||||
SELECT r.id AS "id!: i64",
|
||||
r.kind AS "kind!: String",
|
||||
r.audience AS "audience!: String",
|
||||
r.path AS "path!: String",
|
||||
r.policy_id,
|
||||
p.name AS "policy_name!: String"
|
||||
FROM roots r
|
||||
JOIN policies p ON p.id = r.policy_id
|
||||
ORDER BY r.kind, r.audience
|
||||
"#
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
/// A library root and the policy it carries (§4, §5.1).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Root {
|
||||
pub id: i64,
|
||||
/// `movie` or `tv`.
|
||||
pub kind: String,
|
||||
/// `main` or `kids`.
|
||||
pub audience: String,
|
||||
pub path: String,
|
||||
pub policy_id: i64,
|
||||
pub policy_name: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Db, MIGRATOR};
|
||||
use sqlx::Row;
|
||||
|
||||
/// A migrated database in a directory that lives as long as the guard.
|
||||
async fn fresh() -> (tempfile::TempDir, Db) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = Db::connect(dir.path().join("arr.db"))
|
||||
.await
|
||||
.expect("connect");
|
||||
db.migrate().await.expect("migrate");
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migrations_apply_from_empty() {
|
||||
let (_dir, db) = fresh().await;
|
||||
|
||||
let applied = MIGRATOR.iter().count();
|
||||
let recorded: i64 = sqlx::query_scalar("SELECT count(*) FROM _sqlx_migrations")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.expect("count migrations");
|
||||
|
||||
assert_eq!(recorded, i64::try_from(applied).expect("fits"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migrations_are_idempotent() {
|
||||
let (_dir, db) = fresh().await;
|
||||
db.migrate().await.expect("second run is a no-op");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wal_and_foreign_keys_are_on() {
|
||||
let (_dir, db) = fresh().await;
|
||||
|
||||
let journal: String = sqlx::query_scalar("PRAGMA journal_mode")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.expect("journal_mode");
|
||||
assert_eq!(journal.to_lowercase(), "wal");
|
||||
|
||||
let foreign_keys: i64 = sqlx::query_scalar("PRAGMA foreign_keys")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.expect("foreign_keys");
|
||||
assert_eq!(foreign_keys, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn seeds_two_movie_roots_with_distinct_policies() {
|
||||
let (_dir, db) = fresh().await;
|
||||
|
||||
let roots = db.list_roots().await.expect("list roots");
|
||||
|
||||
assert_eq!(roots.len(), 2, "§5.1: two movie roots, TV comes with #34");
|
||||
assert!(roots.iter().all(|r| r.kind == "movie"));
|
||||
assert_eq!(roots[0].audience, "kids");
|
||||
assert_eq!(roots[0].path, "/mnt/media/movies/kids");
|
||||
assert_eq!(roots[1].audience, "main");
|
||||
assert_eq!(roots[1].path, "/mnt/media/movies/main");
|
||||
assert_ne!(roots[0].policy_id, roots[1].policy_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kids_policy_blacklists_brazilian_dubs_and_prefers_web_dl() {
|
||||
let (_dir, db) = fresh().await;
|
||||
|
||||
let row = sqlx::query(
|
||||
"SELECT p.dub_blacklist, p.source_weights
|
||||
FROM policies p JOIN roots r ON r.policy_id = p.id
|
||||
WHERE r.audience = 'kids'",
|
||||
)
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.expect("kids policy");
|
||||
|
||||
let dub_blacklist: String = row.get(0);
|
||||
let source_weights: String = row.get(1);
|
||||
|
||||
assert!(dub_blacklist.contains("pt-BR"), "§5.2");
|
||||
// §5.2: European Portuguese dubs are a streaming artefact.
|
||||
let webdl = source_weights.find("WEB-DL").expect("WEB-DL weight");
|
||||
let remux = source_weights.find("Remux").expect("Remux weight");
|
||||
assert!(
|
||||
webdl < remux,
|
||||
"WEB-DL is listed first for kids: {source_weights}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreign_keys_are_enforced() {
|
||||
let (_dir, db) = fresh().await;
|
||||
|
||||
let err = sqlx::query("INSERT INTO movies (tmdb_id, title, root_id) VALUES (1, 'x', 999)")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect_err("root 999 does not exist");
|
||||
|
||||
assert!(
|
||||
err.to_string().to_lowercase().contains("foreign key"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn movie_state_is_constrained() {
|
||||
let (_dir, db) = fresh().await;
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO movies (tmdb_id, title, root_id, state)
|
||||
SELECT 693134, 'Dune Part Two', id, 'nonsense' FROM roots WHERE audience = 'main'",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect_err("state is a closed set");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user