fix(dl): resolve download links in arr (#100)
Transmission has no route to Prowlarr, and Prowlarr redirects a .torrent link to a magnet that a file fetch cannot follow. arr fetches the link itself and sends Transmission a magnet or the torrent bytes. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Generated
+1
@@ -88,6 +88,7 @@ dependencies = [
|
|||||||
"arr-parse",
|
"arr-parse",
|
||||||
"arr-probe",
|
"arr-probe",
|
||||||
"axum",
|
"axum",
|
||||||
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"include_dir",
|
"include_dir",
|
||||||
"mime_guess",
|
"mime_guess",
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ tracing = { workspace = true }
|
|||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
base64 = { workspace = true }
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
wiremock = { workspace = true }
|
wiremock = { workspace = true }
|
||||||
|
|
||||||
|
|||||||
+166
-31
@@ -22,7 +22,7 @@ use arr_core::policy::{evaluate, Candidate};
|
|||||||
use arr_core::{score::score, Language, Policy, TitleOverrides, Verdict};
|
use arr_core::{score::score, Language, Policy, TitleOverrides, Verdict};
|
||||||
use arr_db::{blacklist, Blacklist, Db, MoviePolicy};
|
use arr_db::{blacklist, Blacklist, Db, MoviePolicy};
|
||||||
use arr_dl::{AddTorrent, TorrentSource, TransmissionClient};
|
use arr_dl::{AddTorrent, TorrentSource, TransmissionClient};
|
||||||
use arr_indexer::{ProwlarrClient, SearchRelease, SearchRequest};
|
use arr_indexer::{Download, ProwlarrClient, SearchRelease, SearchRequest};
|
||||||
use arr_meta::TmdbClient;
|
use arr_meta::TmdbClient;
|
||||||
|
|
||||||
use crate::indexers::{DiscoveryError, IndexerDirectory};
|
use crate::indexers::{DiscoveryError, IndexerDirectory};
|
||||||
@@ -78,6 +78,8 @@ pub enum GrabError {
|
|||||||
InvalidTmdbId(i64),
|
InvalidTmdbId(i64),
|
||||||
#[error("transmission: {0}")]
|
#[error("transmission: {0}")]
|
||||||
Transmission(#[from] arr_dl::Error),
|
Transmission(#[from] arr_dl::Error),
|
||||||
|
#[error("download link: {0}")]
|
||||||
|
Download(#[from] arr_indexer::DownloadError),
|
||||||
#[error("release {name}: {source}")]
|
#[error("release {name}: {source}")]
|
||||||
Parsed {
|
Parsed {
|
||||||
name: String,
|
name: String,
|
||||||
@@ -105,8 +107,8 @@ impl GrabAction {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
indexers: IndexerDirectory::new(prowlarr.clone()),
|
indexers: IndexerDirectory::new(prowlarr.clone()),
|
||||||
|
grabber: Grabber::new(prowlarr.clone(), transmission, download_dir, seeding),
|
||||||
prowlarr,
|
prowlarr,
|
||||||
grabber: Grabber::new(transmission, download_dir, seeding),
|
|
||||||
tmdb: None,
|
tmdb: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -380,6 +382,9 @@ impl GrabAction {
|
|||||||
/// release onward they are the same writes, so they share this.
|
/// release onward they are the same writes, so they share this.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) struct Grabber {
|
pub(crate) struct Grabber {
|
||||||
|
/// Resolves the winner's indexer link before it is sent on: Transmission
|
||||||
|
/// cannot reach Prowlarr and arr can (issue #100).
|
||||||
|
prowlarr: ProwlarrClient,
|
||||||
transmission: TransmissionClient,
|
transmission: TransmissionClient,
|
||||||
download_dir: PathBuf,
|
download_dir: PathBuf,
|
||||||
seeding: SeedingRules,
|
seeding: SeedingRules,
|
||||||
@@ -441,11 +446,13 @@ impl GrabScope {
|
|||||||
|
|
||||||
impl Grabber {
|
impl Grabber {
|
||||||
pub(crate) fn new(
|
pub(crate) fn new(
|
||||||
|
prowlarr: ProwlarrClient,
|
||||||
transmission: TransmissionClient,
|
transmission: TransmissionClient,
|
||||||
download_dir: PathBuf,
|
download_dir: PathBuf,
|
||||||
seeding: SeedingRules,
|
seeding: SeedingRules,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
prowlarr,
|
||||||
transmission,
|
transmission,
|
||||||
download_dir,
|
download_dir,
|
||||||
seeding,
|
seeding,
|
||||||
@@ -553,6 +560,30 @@ impl Grabber {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolve the winner's indexer link and add it to Transmission.
|
||||||
|
///
|
||||||
|
/// The link is resolved here rather than passed on, because Transmission
|
||||||
|
/// has no route to Prowlarr and cannot follow its redirect to a magnet
|
||||||
|
/// (issue #100).
|
||||||
|
async fn send_to_transmission(
|
||||||
|
&self,
|
||||||
|
winner: &Eligible,
|
||||||
|
loaded: &MoviePolicy,
|
||||||
|
) -> Result<arr_dl::AddedTorrent, GrabError> {
|
||||||
|
let seeding = self.seeding.for_indexer(winner.indexer_id);
|
||||||
|
let source = torrent_source(self.prowlarr.download(&winner.download_url).await?);
|
||||||
|
Ok(self
|
||||||
|
.transmission
|
||||||
|
.add_torrent(AddTorrent {
|
||||||
|
source,
|
||||||
|
label: label(loaded),
|
||||||
|
download_dir: self.download_dir.clone(),
|
||||||
|
seed_ratio_limit: seeding.ratio,
|
||||||
|
seed_idle_limit_minutes: seeding.idle_minutes,
|
||||||
|
})
|
||||||
|
.await?)
|
||||||
|
}
|
||||||
|
|
||||||
/// Add the winning release to Transmission and record the grab.
|
/// Add the winning release to Transmission and record the grab.
|
||||||
///
|
///
|
||||||
/// The search still counts as an attempt (§6.2) on every exit that is
|
/// The search still counts as an attempt (§6.2) on every exit that is
|
||||||
@@ -567,22 +598,11 @@ impl Grabber {
|
|||||||
blacklist: &Blacklist,
|
blacklist: &Blacklist,
|
||||||
winner: Eligible,
|
winner: Eligible,
|
||||||
) -> Result<Option<Outcome>, GrabError> {
|
) -> Result<Option<Outcome>, GrabError> {
|
||||||
let seeding = self.seeding.for_indexer(winner.indexer_id);
|
let added = match self.send_to_transmission(&winner, loaded).await {
|
||||||
let added = match self
|
|
||||||
.transmission
|
|
||||||
.add_torrent(AddTorrent {
|
|
||||||
source: torrent_source(&winner.download_url),
|
|
||||||
label: label(loaded),
|
|
||||||
download_dir: self.download_dir.clone(),
|
|
||||||
seed_ratio_limit: seeding.ratio,
|
|
||||||
seed_idle_limit_minutes: seeding.idle_minutes,
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(added) => added,
|
Ok(added) => added,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
self.record_attempt(database, target).await?;
|
self.record_attempt(database, target).await?;
|
||||||
return Err(error.into());
|
return Err(error);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let infohash = added.hash.to_ascii_lowercase();
|
let infohash = added.hash.to_ascii_lowercase();
|
||||||
@@ -1120,13 +1140,11 @@ pub(crate) fn label_for_root(kind: &str, audience: &str) -> String {
|
|||||||
format!("{kind}-{audience}")
|
format!("{kind}-{audience}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Indexers hand out magnets and `.torrent` links interchangeably;
|
/// A resolved download, in the shape Transmission takes it.
|
||||||
/// Transmission takes either in the same field.
|
fn torrent_source(download: Download) -> TorrentSource {
|
||||||
fn torrent_source(download_url: &str) -> TorrentSource {
|
match download {
|
||||||
if download_url.starts_with("magnet:") {
|
Download::Magnet(uri) => TorrentSource::Magnet(uri),
|
||||||
TorrentSource::Magnet(download_url.to_owned())
|
Download::Torrent(bytes) => TorrentSource::Metainfo(bytes),
|
||||||
} else {
|
|
||||||
TorrentSource::Url(download_url.to_owned())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1151,9 +1169,61 @@ fn rfc3339(time: SystemTime) -> Option<String> {
|
|||||||
chrono::DateTime::from_timestamp(seconds, 0).map(|date| date.to_rfc3339())
|
chrono::DateTime::from_timestamp(seconds, 0).map(|date| date.to_rfc3339())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fixture support for the download-link resolve step (issue #100). Grab
|
||||||
|
/// fixtures carry indexer links, and every lane now fetches them, so the mock
|
||||||
|
/// indexer has to answer the way Prowlarr does.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
|
pub(crate) mod test_downloads {
|
||||||
|
use wiremock::matchers::{method, path_regex};
|
||||||
|
use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
|
||||||
|
|
||||||
|
/// Where rewritten fixture links point on the mock indexer.
|
||||||
|
const PREFIX: &str = "/dl/";
|
||||||
|
|
||||||
|
/// Rewrites the download links in a recorded feed onto `server`.
|
||||||
|
pub(crate) fn rewrite(feed: &str, links: &str, server: &MockServer) -> String {
|
||||||
|
feed.replace(links, &format!("{}{PREFIX}", server.uri()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Prowlarr's live behaviour: the download endpoint 302s to a magnet.
|
||||||
|
pub(crate) async fn mount(server: &MockServer) {
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path_regex(format!("^{PREFIX}")))
|
||||||
|
.respond_with(MagnetRedirect)
|
||||||
|
.mount(server)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct MagnetRedirect;
|
||||||
|
|
||||||
|
impl Respond for MagnetRedirect {
|
||||||
|
fn respond(&self, request: &Request) -> ResponseTemplate {
|
||||||
|
let name = request
|
||||||
|
.url
|
||||||
|
.path()
|
||||||
|
.rsplit('/')
|
||||||
|
.next()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_owned();
|
||||||
|
// Deterministic stand-in for the infohash the tracker would give,
|
||||||
|
// and distinct per release so duplicates still collapse.
|
||||||
|
let mut hash: u64 = 5381;
|
||||||
|
for byte in name.as_bytes() {
|
||||||
|
hash = hash.wrapping_mul(33) ^ u64::from(*byte);
|
||||||
|
}
|
||||||
|
ResponseTemplate::new(302).insert_header(
|
||||||
|
"location",
|
||||||
|
format!("magnet:?xt=urn:btih:{hash:040x}&dn={name}"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[allow(clippy::unwrap_used)]
|
#[allow(clippy::unwrap_used)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use base64::Engine as _;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -1192,8 +1262,11 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn add(&self, arguments: &Value) -> ResponseTemplate {
|
fn add(&self, arguments: &Value) -> ResponseTemplate {
|
||||||
|
// A magnet arrives as `filename`, a torrent body as base64
|
||||||
|
// `metainfo` — either identifies the torrent for the fake.
|
||||||
let source = arguments["filename"]
|
let source = arguments["filename"]
|
||||||
.as_str()
|
.as_str()
|
||||||
|
.or_else(|| arguments["metainfo"].as_str())
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_owned();
|
.to_owned();
|
||||||
let labels: Vec<String> = arguments["labels"]
|
let labels: Vec<String> = arguments["labels"]
|
||||||
@@ -1317,13 +1390,21 @@ mod tests {
|
|||||||
|
|
||||||
async fn prowlarr() -> MockServer {
|
async fn prowlarr() -> MockServer {
|
||||||
let server = MockServer::start().await;
|
let server = MockServer::start().await;
|
||||||
|
let feed = test_downloads::rewrite(RSS, "https://tracker/", &server);
|
||||||
|
test_downloads::mount(&server).await;
|
||||||
|
mount_indexer(&server, &feed).await;
|
||||||
|
server
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One indexer advertising a text search, serving `feed` to every query.
|
||||||
|
async fn mount_indexer(server: &MockServer, feed: &str) {
|
||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/api/v1/indexer"))
|
.and(path("/api/v1/indexer"))
|
||||||
.respond_with(
|
.respond_with(
|
||||||
ResponseTemplate::new(200)
|
ResponseTemplate::new(200)
|
||||||
.set_body_json(json!([{"id": 7, "name": "tracker", "enable": true}])),
|
.set_body_json(json!([{"id": 7, "name": "tracker", "enable": true}])),
|
||||||
)
|
)
|
||||||
.mount(&server)
|
.mount(server)
|
||||||
.await;
|
.await;
|
||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/7/api"))
|
.and(path("/7/api"))
|
||||||
@@ -1331,15 +1412,14 @@ mod tests {
|
|||||||
.respond_with(ResponseTemplate::new(200).set_body_string(
|
.respond_with(ResponseTemplate::new(200).set_body_string(
|
||||||
r#"<caps><searching><search available="yes" supportedParams="q"/></searching></caps>"#,
|
r#"<caps><searching><search available="yes" supportedParams="q"/></searching></caps>"#,
|
||||||
))
|
))
|
||||||
.mount(&server)
|
.mount(server)
|
||||||
.await;
|
.await;
|
||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/7/api"))
|
.and(path("/7/api"))
|
||||||
.and(query_param("t", "search"))
|
.and(query_param("t", "search"))
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_string(RSS))
|
.respond_with(ResponseTemplate::new(200).set_body_string(feed))
|
||||||
.mount(&server)
|
.mount(server)
|
||||||
.await;
|
.await;
|
||||||
server
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn empty_prowlarr() -> MockServer {
|
async fn empty_prowlarr() -> MockServer {
|
||||||
@@ -2004,15 +2084,70 @@ mod tests {
|
|||||||
assert!(fake.torrents().is_empty());
|
assert!(fake.torrents().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// §100: Transmission has no route to the indexer, so a download link
|
||||||
|
/// that answers with the torrent itself is forwarded as inline metainfo
|
||||||
|
/// and the link never leaves arr.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_torrent_body_is_forwarded_inline_and_the_link_never_leaves_arr() {
|
||||||
|
const TORRENT: &[u8] = b"d4:infod6:lengthi1e4:name8:good.mkvee";
|
||||||
|
|
||||||
|
let (_dir, database) = wanted_movie().await;
|
||||||
|
let indexer = MockServer::start().await;
|
||||||
|
let feed = RSS.replace("https://tracker/", &format!("{}/dl/", indexer.uri()));
|
||||||
|
mount_indexer(&indexer, &feed).await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/dl/good.torrent"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200).set_body_raw(TORRENT, "application/x-bittorrent"),
|
||||||
|
)
|
||||||
|
.mount(&indexer)
|
||||||
|
.await;
|
||||||
|
let (downloader, fake) = transmission().await;
|
||||||
|
|
||||||
|
action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
let sent = base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(&fake.torrents()[0].source)
|
||||||
|
.expect("the torrent is sent as base64 metainfo");
|
||||||
|
assert_eq!(sent, TORRENT);
|
||||||
|
let fetched_with_key = indexer
|
||||||
|
.received_requests()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|request| request.url.path() == "/dl/good.torrent")
|
||||||
|
.all(|request| request.headers.contains_key("x-api-key"));
|
||||||
|
assert!(fetched_with_key, "arr resolves the link with its own key");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A link that redirects to a magnet — Prowlarr's usual answer — reaches
|
||||||
|
/// Transmission as the magnet, which it can act on without the indexer.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_link_that_redirects_to_a_magnet_is_sent_as_the_magnet() {
|
||||||
|
let (_dir, database) = wanted_movie().await;
|
||||||
|
let indexer = prowlarr().await;
|
||||||
|
let (downloader, fake) = transmission().await;
|
||||||
|
|
||||||
|
action(&indexer, &downloader).tick(&database).await.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
fake.torrents()[0]
|
||||||
|
.source
|
||||||
|
.starts_with("magnet:?xt=urn:btih:"),
|
||||||
|
"{}",
|
||||||
|
fake.torrents()[0].source
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn indexer_links_and_magnets_both_reach_transmission() {
|
fn a_resolved_download_keeps_its_shape() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
torrent_source("magnet:?xt=urn:btih:abc"),
|
torrent_source(Download::Magnet("magnet:?xt=urn:btih:abc".into())),
|
||||||
TorrentSource::Magnet("magnet:?xt=urn:btih:abc".into())
|
TorrentSource::Magnet("magnet:?xt=urn:btih:abc".into())
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
torrent_source("https://tracker/good.torrent"),
|
torrent_source(Download::Torrent(b"d4:infodee".to_vec())),
|
||||||
TorrentSource::Url("https://tracker/good.torrent".into())
|
TorrentSource::Metainfo(b"d4:infodee".to_vec())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,8 +44,8 @@ impl RssAction {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
indexers: IndexerDirectory::new(prowlarr.clone()),
|
indexers: IndexerDirectory::new(prowlarr.clone()),
|
||||||
|
grabber: Grabber::new(prowlarr.clone(), transmission, download_dir, seeding),
|
||||||
prowlarr,
|
prowlarr,
|
||||||
grabber: Grabber::new(transmission, download_dir, seeding),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,7 +304,7 @@ mod tests {
|
|||||||
use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
|
use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::grab::SeedingLimits;
|
use crate::grab::{test_downloads, SeedingLimits};
|
||||||
use arr_dl::TransmissionClient;
|
use arr_dl::TransmissionClient;
|
||||||
|
|
||||||
/// The recorded feed: two releases of one wanted title, one that only a
|
/// The recorded feed: two releases of one wanted title, one that only a
|
||||||
@@ -352,6 +352,8 @@ mod tests {
|
|||||||
/// feed to an empty query.
|
/// feed to an empty query.
|
||||||
async fn prowlarr() -> MockServer {
|
async fn prowlarr() -> MockServer {
|
||||||
let server = MockServer::start().await;
|
let server = MockServer::start().await;
|
||||||
|
let feed = test_downloads::rewrite(FEED, "https://indexer.invalid/download/", &server);
|
||||||
|
test_downloads::mount(&server).await;
|
||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/api/v1/indexer"))
|
.and(path("/api/v1/indexer"))
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!([
|
.respond_with(ResponseTemplate::new(200).set_body_json(json!([
|
||||||
@@ -374,7 +376,7 @@ mod tests {
|
|||||||
.and(path(format!("/{id}/api")))
|
.and(path(format!("/{id}/api")))
|
||||||
.and(query_param("t", "search"))
|
.and(query_param("t", "search"))
|
||||||
.and(query_param_is_missing("q"))
|
.and(query_param_is_missing("q"))
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_string(FEED))
|
.respond_with(ResponseTemplate::new(200).set_body_string(&feed))
|
||||||
.mount(&server)
|
.mount(&server)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ impl TvGrabAction {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
indexers: IndexerDirectory::new(prowlarr.clone()),
|
indexers: IndexerDirectory::new(prowlarr.clone()),
|
||||||
|
grabber: Grabber::new(prowlarr.clone(), transmission, download_dir, seeding),
|
||||||
prowlarr,
|
prowlarr,
|
||||||
grabber: Grabber::new(transmission, download_dir, seeding),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -678,7 +678,7 @@ mod tests {
|
|||||||
use wiremock::matchers::{method, path, query_param};
|
use wiremock::matchers::{method, path, query_param};
|
||||||
use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
|
use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
|
||||||
|
|
||||||
use crate::grab::SeedingLimits;
|
use crate::grab::{test_downloads, SeedingLimits};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
@@ -786,6 +786,8 @@ mod tests {
|
|||||||
|
|
||||||
async fn prowlarr() -> MockServer {
|
async fn prowlarr() -> MockServer {
|
||||||
let server = MockServer::start().await;
|
let server = MockServer::start().await;
|
||||||
|
let feed = test_downloads::rewrite(TV_RSS, "https://tracker/", &server);
|
||||||
|
test_downloads::mount(&server).await;
|
||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/api/v1/indexer"))
|
.and(path("/api/v1/indexer"))
|
||||||
.respond_with(
|
.respond_with(
|
||||||
@@ -805,7 +807,7 @@ mod tests {
|
|||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/7/api"))
|
.and(path("/7/api"))
|
||||||
.and(query_param("t", "search"))
|
.and(query_param("t", "search"))
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_string(TV_RSS))
|
.respond_with(ResponseTemplate::new(200).set_body_string(feed))
|
||||||
.mount(&server)
|
.mount(&server)
|
||||||
.await;
|
.await;
|
||||||
server
|
server
|
||||||
|
|||||||
@@ -16,12 +16,14 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
|
|||||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
/// Where Transmission is to get the torrent from.
|
/// Where Transmission is to get the torrent from.
|
||||||
|
///
|
||||||
|
/// Deliberately not an indexer URL: Transmission has no route to Prowlarr and
|
||||||
|
/// cannot follow its redirect to a magnet, so arr resolves the link itself and
|
||||||
|
/// hands over the result (issue #100).
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub enum TorrentSource {
|
pub enum TorrentSource {
|
||||||
Magnet(String),
|
Magnet(String),
|
||||||
/// An HTTP link to a `.torrent`, fetched by Transmission itself. Indexer
|
/// The bytes of a `.torrent` file, sent inline.
|
||||||
/// download links arrive in this form.
|
|
||||||
Url(String),
|
|
||||||
Metainfo(Vec<u8>),
|
Metainfo(Vec<u8>),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,7 +164,7 @@ impl TransmissionClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
match request.source {
|
match request.source {
|
||||||
TorrentSource::Magnet(uri) | TorrentSource::Url(uri) => {
|
TorrentSource::Magnet(uri) => {
|
||||||
arguments["filename"] = json!(uri);
|
arguments["filename"] = json!(uri);
|
||||||
}
|
}
|
||||||
TorrentSource::Metainfo(bytes) => {
|
TorrentSource::Metainfo(bytes) => {
|
||||||
|
|||||||
@@ -7,10 +7,14 @@ pub use search::{IndexerSearch, SearchError, SearchRelease, SearchRequest, TvSel
|
|||||||
use std::{collections::BTreeSet, time::Duration};
|
use std::{collections::BTreeSet, time::Duration};
|
||||||
|
|
||||||
use quick_xml::events::Event;
|
use quick_xml::events::Event;
|
||||||
use reqwest::{Client, StatusCode};
|
use reqwest::{header::LOCATION, Client, StatusCode, Url};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// How many redirects a download link may take before arr gives up. Prowlarr
|
||||||
|
/// takes one hop to a magnet or to the tracker's own file; the rest is slack.
|
||||||
|
const MAX_DOWNLOAD_REDIRECTS: usize = 5;
|
||||||
|
|
||||||
/// A Prowlarr indexer and the Torznab capabilities it advertises.
|
/// A Prowlarr indexer and the Torznab capabilities it advertises.
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct Indexer {
|
pub struct Indexer {
|
||||||
@@ -62,6 +66,9 @@ pub struct ProwlarrClient {
|
|||||||
base_url: String,
|
base_url: String,
|
||||||
api_key: String,
|
api_key: String,
|
||||||
client: Client,
|
client: Client,
|
||||||
|
/// Kept apart from `client` because resolving a download link depends on
|
||||||
|
/// seeing the redirect rather than following it (§7.1, issue #100).
|
||||||
|
downloads: Client,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProwlarrClient {
|
impl ProwlarrClient {
|
||||||
@@ -75,11 +82,17 @@ impl ProwlarrClient {
|
|||||||
.timeout(Duration::from_secs(30))
|
.timeout(Duration::from_secs(30))
|
||||||
.build()
|
.build()
|
||||||
.map_err(|_| Error::Client)?;
|
.map_err(|_| Error::Client)?;
|
||||||
|
let downloads = Client::builder()
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
|
.build()
|
||||||
|
.map_err(|_| Error::Client)?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
base_url: base_url.into().trim_end_matches('/').to_owned(),
|
base_url: base_url.into().trim_end_matches('/').to_owned(),
|
||||||
api_key: api_key.into(),
|
api_key: api_key.into(),
|
||||||
client,
|
client,
|
||||||
|
downloads,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,6 +160,101 @@ impl ProwlarrClient {
|
|||||||
})?;
|
})?;
|
||||||
parse_capabilities(&body)
|
parse_capabilities(&body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolves an indexer download link into something the download client
|
||||||
|
/// can take without reaching the indexer itself.
|
||||||
|
///
|
||||||
|
/// Handing Transmission the Prowlarr link fails twice over: Transmission
|
||||||
|
/// has no route to Prowlarr, and Prowlarr answers a `.torrent` link with a
|
||||||
|
/// redirect to a magnet, which a plain file fetch cannot follow. arr has
|
||||||
|
/// the route and the API key, so it resolves the link here and passes on
|
||||||
|
/// the result (issue #100).
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Returns an error when the link cannot be fetched or resolves to
|
||||||
|
/// something that is neither a magnet nor a torrent.
|
||||||
|
pub async fn download(&self, download_url: &str) -> Result<Download, DownloadError> {
|
||||||
|
if let Some(magnet) = as_magnet(download_url) {
|
||||||
|
return Ok(Download::Magnet(magnet));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut url = Url::parse(download_url).map_err(|_| DownloadError::InvalidUrl)?;
|
||||||
|
for _ in 0..MAX_DOWNLOAD_REDIRECTS {
|
||||||
|
let mut request = self.downloads.get(url.clone());
|
||||||
|
// Only ever to Prowlarr itself: a redirect can leave for a tracker
|
||||||
|
// and the key must not go with it.
|
||||||
|
if self.is_prowlarr(&url) {
|
||||||
|
request = request.header("X-Api-Key", &self.api_key);
|
||||||
|
}
|
||||||
|
let response = request
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|error| DownloadError::Request {
|
||||||
|
status: error.status(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if response.status().is_redirection() {
|
||||||
|
let location = response
|
||||||
|
.headers()
|
||||||
|
.get(LOCATION)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or(DownloadError::UnusableRedirect)?;
|
||||||
|
if let Some(magnet) = as_magnet(location) {
|
||||||
|
return Ok(Download::Magnet(magnet));
|
||||||
|
}
|
||||||
|
url = url
|
||||||
|
.join(location)
|
||||||
|
.map_err(|_| DownloadError::UnusableRedirect)?;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = response
|
||||||
|
.error_for_status()
|
||||||
|
.map_err(|error| DownloadError::Request {
|
||||||
|
status: error.status(),
|
||||||
|
})?
|
||||||
|
.bytes()
|
||||||
|
.await
|
||||||
|
.map_err(|error| DownloadError::Request {
|
||||||
|
status: error.status(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Some indexers answer with the magnet as the body rather than as
|
||||||
|
// a redirect.
|
||||||
|
if let Some(magnet) = std::str::from_utf8(&body).ok().and_then(as_magnet) {
|
||||||
|
return Ok(Download::Magnet(magnet));
|
||||||
|
}
|
||||||
|
// A torrent is a bencoded dictionary. An HTML error page or a
|
||||||
|
// login form must not reach the download client as metainfo.
|
||||||
|
if body.first() != Some(&b'd') {
|
||||||
|
return Err(DownloadError::NotATorrent);
|
||||||
|
}
|
||||||
|
return Ok(Download::Torrent(body.to_vec()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(DownloadError::TooManyRedirects(MAX_DOWNLOAD_REDIRECTS))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_prowlarr(&self, url: &Url) -> bool {
|
||||||
|
Url::parse(&self.base_url).is_ok_and(|base| base.origin() == url.origin())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What an indexer's download link resolved to.
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub enum Download {
|
||||||
|
Magnet(String),
|
||||||
|
/// The bytes of a `.torrent` file.
|
||||||
|
Torrent(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn as_magnet(value: &str) -> Option<String> {
|
||||||
|
let value = value.trim();
|
||||||
|
value
|
||||||
|
.get(..7)
|
||||||
|
.is_some_and(|scheme| scheme.eq_ignore_ascii_case("magnet:"))
|
||||||
|
.then(|| value.to_owned())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Errors returned while discovering Prowlarr indexers.
|
/// Errors returned while discovering Prowlarr indexers.
|
||||||
@@ -160,6 +268,21 @@ pub enum Error {
|
|||||||
Response,
|
Response,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A download link that could not be turned into a magnet or a torrent.
|
||||||
|
#[derive(Clone, Debug, Eq, Error, PartialEq)]
|
||||||
|
pub enum DownloadError {
|
||||||
|
#[error("download link request failed (status: {status:?})")]
|
||||||
|
Request { status: Option<StatusCode> },
|
||||||
|
#[error("download link is not a valid URL")]
|
||||||
|
InvalidUrl,
|
||||||
|
#[error("download link redirected to a location that cannot be used")]
|
||||||
|
UnusableRedirect,
|
||||||
|
#[error("download link returned neither a torrent nor a magnet")]
|
||||||
|
NotATorrent,
|
||||||
|
#[error("download link still redirected after {0} hops")]
|
||||||
|
TooManyRedirects(usize),
|
||||||
|
}
|
||||||
|
|
||||||
/// A failed capabilities request for an otherwise usable Prowlarr indexer.
|
/// A failed capabilities request for an otherwise usable Prowlarr indexer.
|
||||||
#[derive(Clone, Debug, Eq, Error, PartialEq)]
|
#[derive(Clone, Debug, Eq, Error, PartialEq)]
|
||||||
pub enum CapabilityError {
|
pub enum CapabilityError {
|
||||||
@@ -290,8 +413,8 @@ fn is_available(value: &str) -> bool {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
parse_capabilities, Capabilities, CapabilityError, ProwlarrClient, SearchRequest,
|
parse_capabilities, Capabilities, CapabilityError, Download, DownloadError, ProwlarrClient,
|
||||||
TvSelector, TvTarget,
|
SearchRequest, TvSelector, TvTarget,
|
||||||
};
|
};
|
||||||
use reqwest::StatusCode;
|
use reqwest::StatusCode;
|
||||||
use wiremock::{
|
use wiremock::{
|
||||||
@@ -301,6 +424,10 @@ mod tests {
|
|||||||
|
|
||||||
const API_KEY: &str = "test-api-key";
|
const API_KEY: &str = "test-api-key";
|
||||||
|
|
||||||
|
fn client(server: &MockServer) -> ProwlarrClient {
|
||||||
|
ProwlarrClient::new(server.uri(), API_KEY).expect("HTTP client can be created")
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn enumerates_indexers_with_recorded_capabilities() {
|
async fn enumerates_indexers_with_recorded_capabilities() {
|
||||||
let server = MockServer::start().await;
|
let server = MockServer::start().await;
|
||||||
@@ -369,6 +496,145 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// #100: Prowlarr answers a `.torrent` link with a redirect to a magnet.
|
||||||
|
/// The download client cannot follow that, so arr does it here.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_redirect_to_a_magnet_resolves_to_the_magnet() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/download"))
|
||||||
|
.and(header("X-Api-Key", API_KEY))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(302)
|
||||||
|
.insert_header("location", "magnet:?xt=urn:btih:abc&dn=release"),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let download = client(&server)
|
||||||
|
.download(&format!("{}/download", server.uri()))
|
||||||
|
.await
|
||||||
|
.expect("the redirect resolves");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
download,
|
||||||
|
Download::Magnet("magnet:?xt=urn:btih:abc&dn=release".to_owned())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_torrent_body_resolves_to_its_bytes() {
|
||||||
|
let body: &[u8] = b"d4:infod6:lengthi1e4:name7:one.mkvee";
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/download"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_raw(body, "application/x-bittorrent"))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let download = client(&server)
|
||||||
|
.download(&format!("{}/download", server.uri()))
|
||||||
|
.await
|
||||||
|
.expect("the body is a torrent");
|
||||||
|
|
||||||
|
assert_eq!(download, Download::Torrent(body.to_vec()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A magnet in the feed needs no request at all.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_magnet_link_resolves_without_a_request() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
|
||||||
|
let download = client(&server)
|
||||||
|
.download("magnet:?xt=urn:btih:abc")
|
||||||
|
.await
|
||||||
|
.expect("a magnet is already resolved");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
download,
|
||||||
|
Download::Magnet("magnet:?xt=urn:btih:abc".to_owned())
|
||||||
|
);
|
||||||
|
assert!(server
|
||||||
|
.received_requests()
|
||||||
|
.await
|
||||||
|
.expect("recorded")
|
||||||
|
.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The API key belongs to Prowlarr. A redirect that leaves for a tracker
|
||||||
|
/// must not carry it there.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_api_key_never_follows_a_redirect_off_prowlarr() {
|
||||||
|
let tracker = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/file.torrent"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200)
|
||||||
|
.set_body_raw(b"d4:infodee".as_slice(), "application/x-bittorrent"),
|
||||||
|
)
|
||||||
|
.mount(&tracker)
|
||||||
|
.await;
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/download"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(302)
|
||||||
|
.insert_header("location", format!("{}/file.torrent", tracker.uri())),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let download = client(&server)
|
||||||
|
.download(&format!("{}/download", server.uri()))
|
||||||
|
.await
|
||||||
|
.expect("the tracker serves the file");
|
||||||
|
|
||||||
|
assert_eq!(download, Download::Torrent(b"d4:infodee".to_vec()));
|
||||||
|
let sent_key = tracker
|
||||||
|
.received_requests()
|
||||||
|
.await
|
||||||
|
.expect("recorded")
|
||||||
|
.into_iter()
|
||||||
|
.any(|request| request.headers.contains_key("x-api-key"));
|
||||||
|
assert!(!sent_key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An HTML error page or a login form is not a torrent, and passing it on
|
||||||
|
/// as metainfo would only fail later and less clearly.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_body_that_is_not_a_torrent_is_refused() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/download"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string("<html>login</html>"))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let error = client(&server)
|
||||||
|
.download(&format!("{}/download", server.uri()))
|
||||||
|
.await
|
||||||
|
.expect_err("HTML is not a torrent");
|
||||||
|
|
||||||
|
assert_eq!(error, DownloadError::NotATorrent);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_redirect_loop_gives_up() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/download"))
|
||||||
|
.respond_with(ResponseTemplate::new(302).insert_header("location", "/download"))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let error = client(&server)
|
||||||
|
.download(&format!("{}/download", server.uri()))
|
||||||
|
.await
|
||||||
|
.expect_err("the link never resolves");
|
||||||
|
|
||||||
|
assert_eq!(error, DownloadError::TooManyRedirects(5));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn tv_requests_degrade_to_what_each_indexer_accepts() {
|
fn tv_requests_degrade_to_what_each_indexer_accepts() {
|
||||||
let alpha = capabilities(include_str!("../tests/fixtures/alpha-caps.xml"));
|
let alpha = capabilities(include_str!("../tests/fixtures/alpha-caps.xml"));
|
||||||
|
|||||||
Reference in New Issue
Block a user