fix(daemon): bound metadata cost and record every grab attempt
Address review 127 on PR #90: - a failed metadata refresh no longer aborts the whole tick - TMDB refresh is gated by its own persisted TTL, not paid every tick - a search attempt is recorded on every non-grab exit (Transmission error, blacklisted-infohash drop), not only when no winner is found Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+14
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "SQLite",
|
||||
"query": "\n SELECT m.id AS \"id!: i64\",\n m.tmdb_id AS \"tmdb_id!: i64\",\n m.title AS \"title!: String\",\n m.year,\n m.original_language,\n m.search_attempts AS \"search_attempts!: i64\",\n m.last_searched_at\n FROM movies m\n WHERE m.wanted = 1\n AND m.blocked = 0\n AND NOT EXISTS (\n SELECT 1 FROM media_files f\n WHERE f.owner_kind = 'movie' AND f.owner_id = m.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM grabs g\n WHERE g.target_kind = 'movie' AND g.target_id = m.id\n AND g.state IN ('sent', 'downloaded', 'imported')\n )\n ORDER BY m.last_searched_at IS NOT NULL, m.last_searched_at, m.id\n LIMIT ?\n ",
|
||||
"query": "\n SELECT m.id AS \"id!: i64\",\n m.tmdb_id AS \"tmdb_id!: i64\",\n m.title AS \"title!: String\",\n m.year,\n m.original_language,\n m.search_attempts AS \"search_attempts!: i64\",\n m.last_searched_at,\n m.digital_release,\n m.metadata_refreshed_at\n FROM movies m\n WHERE m.wanted = 1\n AND m.blocked = 0\n AND NOT EXISTS (\n SELECT 1 FROM media_files f\n WHERE f.owner_kind = 'movie' AND f.owner_id = m.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM grabs g\n WHERE g.target_kind = 'movie' AND g.target_id = m.id\n AND g.state IN ('sent', 'downloaded', 'imported')\n )\n ORDER BY m.last_searched_at IS NOT NULL, m.last_searched_at, m.id\n LIMIT ?\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -37,6 +37,16 @@
|
||||
"name": "last_searched_at",
|
||||
"ordinal": 6,
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"name": "digital_release",
|
||||
"ordinal": 7,
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"name": "metadata_refreshed_at",
|
||||
"ordinal": 8,
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -49,8 +59,10 @@
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "f891c0c0e70ac2fdb525812329cf31d41d0591ff80ba30b7a8fa0f788567dcd4"
|
||||
"hash": "0db845eb00a34dce18b6a26efc83b99efeddf05aae9a32765680a97b5db73bb0"
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"db_name": "SQLite",
|
||||
"query": "UPDATE movies\n SET title = ?, year = ?, original_language = ?, digital_release = ?,\n search_attempts = 0, last_searched_at = NULL,\n updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')\n WHERE id = ? AND (\n title IS NOT ? OR year IS NOT ? OR original_language IS NOT ?\n OR digital_release IS NOT ?\n )",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Right": 9
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "2ace3eaa48da2158eeb97fa2051b1f88fa9b5bb8e8660d66c91f9cf61ecb1028"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "SQLite",
|
||||
"query": "UPDATE movies\n SET title = ?, year = ?, original_language = ?, digital_release = ?,\n metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'),\n search_attempts = 0, last_searched_at = NULL,\n updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')\n WHERE id = ? AND (\n title IS NOT ? OR year IS NOT ? OR original_language IS NOT ?\n OR digital_release IS NOT ?\n )",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Right": 9
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "9da028029e93a01bd4be5a3b065556e60ba014a782d297aa01ba4b80dc94b485"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "SQLite",
|
||||
"query": "UPDATE movies SET metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')\n WHERE id = ?",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Right": 1
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "de2153eefd6ec03ceeaf640d599271834cb06c94eda7a9956c5bd7399593c0ae"
|
||||
}
|
||||
@@ -151,8 +151,16 @@ impl GrabAction {
|
||||
}
|
||||
|
||||
for movie in gaps {
|
||||
let Some(movie) = self.refresh_metadata(database, movie).await? else {
|
||||
continue;
|
||||
let (movie_id, title) = (movie.id, movie.title.clone());
|
||||
let movie = match self.refresh_metadata(database, movie).await {
|
||||
Ok(Some(movie)) => movie,
|
||||
Ok(None) => continue,
|
||||
// A title's metadata is not the rest of the tick's problem,
|
||||
// same as a grab failure below.
|
||||
Err(error) => {
|
||||
tracing::error!(movie_id, title, %error, "metadata refresh failed");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if !search_due(&movie) {
|
||||
continue;
|
||||
@@ -186,6 +194,10 @@ impl GrabAction {
|
||||
let Some(tmdb) = &self.tmdb else {
|
||||
return Ok(Some(movie));
|
||||
};
|
||||
if !metadata_refresh_due(movie.metadata_refreshed_at.as_deref()) {
|
||||
let released = is_digitally_released(movie.digital_release.as_deref());
|
||||
return Ok(released.then_some(movie));
|
||||
}
|
||||
let tmdb_id =
|
||||
u32::try_from(movie.tmdb_id).map_err(|_| GrabError::InvalidTmdbId(movie.id))?;
|
||||
let metadata = tmdb.movie(tmdb_id).await?;
|
||||
@@ -200,6 +212,7 @@ impl GrabAction {
|
||||
let changed = sqlx::query!(
|
||||
r#"UPDATE movies
|
||||
SET title = ?, year = ?, original_language = ?, digital_release = ?,
|
||||
metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'),
|
||||
search_attempts = 0, last_searched_at = NULL,
|
||||
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
||||
WHERE id = ? AND (
|
||||
@@ -225,6 +238,16 @@ impl GrabAction {
|
||||
movie_id = movie.id,
|
||||
"metadata changed; reset targeted search backoff"
|
||||
);
|
||||
} else {
|
||||
// Still stamp the refresh even when nothing changed, or the TTL
|
||||
// gate above never engages and every tick pays for TMDB again.
|
||||
sqlx::query!(
|
||||
"UPDATE movies SET metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
||||
WHERE id = ?",
|
||||
movie.id
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await?;
|
||||
}
|
||||
if !metadata.is_digitally_released(chrono::Utc::now().date_naive()) {
|
||||
tracing::debug!(
|
||||
@@ -245,6 +268,8 @@ impl GrabAction {
|
||||
} else {
|
||||
movie.last_searched_at
|
||||
},
|
||||
digital_release,
|
||||
metadata_refreshed_at: None,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -450,23 +475,49 @@ impl GrabAction {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
self.send_winner(database, movie, &loaded, &blacklist, winner)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Add the winning release to Transmission and record the grab.
|
||||
///
|
||||
/// The search still counts as an attempt (§6.2) on every exit that is
|
||||
/// not a completed grab — a Transmission error or a blacklisted-infohash
|
||||
/// drop must not leave the same release to repeat next tick with no
|
||||
/// backoff.
|
||||
async fn send_winner(
|
||||
&self,
|
||||
database: &Db,
|
||||
movie: &PendingMovie,
|
||||
loaded: &MoviePolicy,
|
||||
blacklist: &Blacklist,
|
||||
winner: Eligible,
|
||||
) -> Result<Option<Outcome>, GrabError> {
|
||||
let seeding = self.seeding.for_indexer(winner.indexer_id);
|
||||
let added = self
|
||||
let added = match self
|
||||
.transmission
|
||||
.add_torrent(AddTorrent {
|
||||
source: torrent_source(&winner.download_url),
|
||||
label: label(&loaded),
|
||||
label: label(loaded),
|
||||
download_dir: self.download_dir.clone(),
|
||||
seed_ratio_limit: seeding.ratio,
|
||||
seed_idle_limit_minutes: seeding.idle_minutes,
|
||||
})
|
||||
.await?;
|
||||
.await
|
||||
{
|
||||
Ok(added) => added,
|
||||
Err(error) => {
|
||||
record_search(database, movie.id).await?;
|
||||
return Err(error.into());
|
||||
}
|
||||
};
|
||||
let infohash = added.hash.to_ascii_lowercase();
|
||||
|
||||
// §6.3's second key. A `.torrent` link hides its infohash until
|
||||
// Transmission has fetched it, so the same blacklisted torrent can
|
||||
// reach here under a new name.
|
||||
if blacklist.blocks_infohash(&infohash) {
|
||||
record_search(database, movie.id).await?;
|
||||
self.drop_blacklisted_torrent(database, movie, &winner, &added)
|
||||
.await?;
|
||||
return Ok(None);
|
||||
@@ -595,6 +646,8 @@ struct PendingMovie {
|
||||
original_language: Option<String>,
|
||||
search_attempts: i64,
|
||||
last_searched_at: Option<String>,
|
||||
digital_release: Option<String>,
|
||||
metadata_refreshed_at: Option<String>,
|
||||
}
|
||||
|
||||
/// The eligible view of a stored release, ranked for selection.
|
||||
@@ -622,7 +675,9 @@ async fn pending_movies(database: &Db) -> Result<Vec<PendingMovie>, GrabError> {
|
||||
m.year,
|
||||
m.original_language,
|
||||
m.search_attempts AS "search_attempts!: i64",
|
||||
m.last_searched_at
|
||||
m.last_searched_at,
|
||||
m.digital_release,
|
||||
m.metadata_refreshed_at
|
||||
FROM movies m
|
||||
WHERE m.wanted = 1
|
||||
AND m.blocked = 0
|
||||
@@ -653,6 +708,8 @@ async fn pending_movies(database: &Db) -> Result<Vec<PendingMovie>, GrabError> {
|
||||
original_language: row.original_language,
|
||||
search_attempts: row.search_attempts,
|
||||
last_searched_at: row.last_searched_at,
|
||||
digital_release: row.digital_release,
|
||||
metadata_refreshed_at: row.metadata_refreshed_at,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@@ -674,6 +731,25 @@ fn search_due(movie: &PendingMovie) -> bool {
|
||||
last_searched_at.with_timezone(&chrono::Utc) + backoff <= chrono::Utc::now()
|
||||
}
|
||||
|
||||
fn metadata_refresh_due(metadata_refreshed_at: Option<&str>) -> bool {
|
||||
let Some(refreshed_at) = metadata_refreshed_at else {
|
||||
return true;
|
||||
};
|
||||
let Ok(refreshed_at) = chrono::DateTime::parse_from_rfc3339(refreshed_at) else {
|
||||
return true;
|
||||
};
|
||||
// Independent of the search backoff (§6.2): a title stuck on a day-long
|
||||
// backoff, or one with no digital release date yet, must not cost a
|
||||
// TMDB call every tick.
|
||||
refreshed_at.with_timezone(&chrono::Utc) + chrono::TimeDelta::hours(6) <= chrono::Utc::now()
|
||||
}
|
||||
|
||||
fn is_digitally_released(digital_release: Option<&str>) -> bool {
|
||||
digital_release
|
||||
.and_then(|date| date.parse::<chrono::NaiveDate>().ok())
|
||||
.is_some_and(|date| date <= chrono::Utc::now().date_naive())
|
||||
}
|
||||
|
||||
/// Cache the classified release and associate it with the title.
|
||||
///
|
||||
/// Returns the candidate only when the release is eligible: automatic
|
||||
@@ -1499,6 +1575,60 @@ mod tests {
|
||||
assert_eq!(targeted_searches(&indexer).await, 5);
|
||||
}
|
||||
|
||||
/// A title stuck on backoff must not pay for TMDB on every tick: the
|
||||
/// metadata refresh has its own TTL, independent of the search backoff.
|
||||
///
|
||||
/// A fresh `GrabAction` (and so a fresh `TmdbClient`) is built for every
|
||||
/// tick, as a restarted process would, so the only thing that can be
|
||||
/// suppressing a real TMDB request is the persisted
|
||||
/// `metadata_refreshed_at` gate rather than the client's own in-process
|
||||
/// response cache.
|
||||
#[tokio::test]
|
||||
async fn metadata_refresh_is_throttled_by_its_own_ttl() {
|
||||
let (_dir, database) = wanted_movie().await;
|
||||
let indexer = empty_prowlarr().await;
|
||||
let metadata = tmdb(RELEASED_METADATA).await;
|
||||
let (downloader, _fake) = transmission().await;
|
||||
|
||||
action_with_tmdb(&indexer, &downloader, &metadata)
|
||||
.tick(&database)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(metadata.received_requests().await.unwrap().len(), 1);
|
||||
assert_eq!(targeted_searches(&indexer).await, 1);
|
||||
|
||||
// Due for another search attempt, but the metadata refresh is not
|
||||
// due yet: TMDB is not called again, and the stored digital release
|
||||
// still gates the search correctly.
|
||||
sqlx::query(
|
||||
"UPDATE movies SET last_searched_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-2 hours')",
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
action_with_tmdb(&indexer, &downloader, &metadata)
|
||||
.tick(&database)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(metadata.received_requests().await.unwrap().len(), 1);
|
||||
assert_eq!(targeted_searches(&indexer).await, 2);
|
||||
|
||||
// Past the refresh TTL: the next due attempt pays for TMDB again.
|
||||
sqlx::query(
|
||||
"UPDATE movies SET last_searched_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 hours'),
|
||||
metadata_refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 hours')",
|
||||
)
|
||||
.execute(database.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
action_with_tmdb(&indexer, &downloader, &metadata)
|
||||
.tick(&database)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(metadata.received_requests().await.unwrap().len(), 2);
|
||||
assert_eq!(targeted_searches(&indexer).await, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_changes_reset_a_title_backoff() {
|
||||
let (_dir, database) = wanted_movie().await;
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Bounds how often targeted search pays for a TMDB call per movie,
|
||||
-- independent of the search backoff schedule (§6.2): a title on a
|
||||
-- day-long backoff, or one with no digital release date yet, must not
|
||||
-- cost a TMDB call every 30 s tick.
|
||||
ALTER TABLE movies ADD COLUMN metadata_refreshed_at TEXT;
|
||||
Reference in New Issue
Block a user