|
|
|
@@ -6,11 +6,15 @@
|
|
|
|
|
//! the body carries the verdict, so a degraded service can still explain
|
|
|
|
|
//! itself to the UI instead of looking like a fourth outage.
|
|
|
|
|
|
|
|
|
|
use std::ffi::OsStr;
|
|
|
|
|
|
|
|
|
|
use axum::extract::State;
|
|
|
|
|
use axum::Json;
|
|
|
|
|
use serde::Serialize;
|
|
|
|
|
use utoipa::ToSchema;
|
|
|
|
|
|
|
|
|
|
use arr_subs::{binary_present, translate as backend_error};
|
|
|
|
|
|
|
|
|
|
use crate::state::AppState;
|
|
|
|
|
|
|
|
|
|
/// Whether the service as a whole can do its job.
|
|
|
|
@@ -68,6 +72,44 @@ impl Check {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// One enabled subtitle provider's verdict (#200).
|
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)]
|
|
|
|
|
pub struct ProviderCheck {
|
|
|
|
|
/// The provider's id as settings name it.
|
|
|
|
|
pub id: String,
|
|
|
|
|
pub status: Status,
|
|
|
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
|
|
|
pub detail: Option<String>,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The subtitle lane's verdicts (DESIGN.md §15, #200). Only what is actually
|
|
|
|
|
/// in use appears here: providers nobody enabled and engines nobody selected
|
|
|
|
|
/// cannot be broken.
|
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)]
|
|
|
|
|
pub struct SubtitleHealth {
|
|
|
|
|
/// One entry per provider the settings enable, in that order.
|
|
|
|
|
pub providers: Vec<ProviderCheck>,
|
|
|
|
|
/// The selected translation engine, or `None` when none is chosen —
|
|
|
|
|
/// which is a configuration state, not an outage.
|
|
|
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
|
|
|
pub translation: Option<Check>,
|
|
|
|
|
/// The `alass` sync binary, present at its configured path.
|
|
|
|
|
pub alass: Check,
|
|
|
|
|
/// The `ffmpeg` extraction binary, present at its configured path.
|
|
|
|
|
pub ffmpeg: Check,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl SubtitleHealth {
|
|
|
|
|
/// Every verdict the lane carries, for the overall status.
|
|
|
|
|
fn statuses(&self) -> impl Iterator<Item = Status> + '_ {
|
|
|
|
|
self.providers
|
|
|
|
|
.iter()
|
|
|
|
|
.map(|provider| provider.status)
|
|
|
|
|
.chain(self.translation.iter().map(|check| check.status))
|
|
|
|
|
.chain([self.alass.status, self.ffmpeg.status])
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The body of `GET /api/health`.
|
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)]
|
|
|
|
|
pub struct HealthReport {
|
|
|
|
@@ -78,9 +120,11 @@ pub struct HealthReport {
|
|
|
|
|
pub prowlarr: Check,
|
|
|
|
|
pub transmission: Check,
|
|
|
|
|
pub tmdb: Check,
|
|
|
|
|
pub subtitles: SubtitleHealth,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Report reachability of Prowlarr, Transmission and TMDB.
|
|
|
|
|
/// Report reachability of Prowlarr, Transmission and TMDB, plus the
|
|
|
|
|
/// subtitle upstreams (#200).
|
|
|
|
|
#[utoipa::path(
|
|
|
|
|
get,
|
|
|
|
|
path = "/api/health",
|
|
|
|
@@ -90,17 +134,15 @@ pub struct HealthReport {
|
|
|
|
|
),
|
|
|
|
|
)]
|
|
|
|
|
pub async fn health(State(state): State<AppState>) -> Json<HealthReport> {
|
|
|
|
|
// Three independent network probes; serialising them would make the
|
|
|
|
|
// endpoint as slow as the sum of the timeouts.
|
|
|
|
|
let (prowlarr, transmission, tmdb) = tokio::join!(
|
|
|
|
|
probe_prowlarr(&state),
|
|
|
|
|
probe_transmission(&state),
|
|
|
|
|
probe_tmdb(&state),
|
|
|
|
|
);
|
|
|
|
|
// Independent network probes; serialising them would make the endpoint
|
|
|
|
|
// as slow as the sum of the timeouts.
|
|
|
|
|
let (prowlarr, transmission, tmdb, subtitles) =
|
|
|
|
|
tokio::join!(probe_prowlarr(&state), probe_transmission(&state), probe_tmdb(&state), probe_subtitles(&state));
|
|
|
|
|
|
|
|
|
|
let status = if [prowlarr.status, transmission.status, tmdb.status]
|
|
|
|
|
.iter()
|
|
|
|
|
.all(|s| *s == Status::Ok)
|
|
|
|
|
.into_iter()
|
|
|
|
|
.chain(subtitles.statuses())
|
|
|
|
|
.all(|check| check == Status::Ok)
|
|
|
|
|
{
|
|
|
|
|
Health::Ok
|
|
|
|
|
} else {
|
|
|
|
@@ -113,6 +155,7 @@ pub async fn health(State(state): State<AppState>) -> Json<HealthReport> {
|
|
|
|
|
prowlarr,
|
|
|
|
|
transmission,
|
|
|
|
|
tmdb,
|
|
|
|
|
subtitles,
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -181,3 +224,313 @@ async fn probe_tmdb(state: &AppState) -> Check {
|
|
|
|
|
fn describe(err: reqwest::Error) -> String {
|
|
|
|
|
err.without_url().to_string()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---- the subtitle lane (DESIGN.md §15, #200) -----------------------------
|
|
|
|
|
|
|
|
|
|
/// Probe every subtitle lamp: enabled providers, the selected engine, and
|
|
|
|
|
/// the two binaries. The settings row decides what is *in use* — a provider
|
|
|
|
|
/// nobody enabled or an engine nobody selected is not probed at all, so it
|
|
|
|
|
/// cannot fail a lamp.
|
|
|
|
|
async fn probe_subtitles(state: &AppState) -> SubtitleHealth {
|
|
|
|
|
let settings = match state.database() {
|
|
|
|
|
Some(_) => crate::subtitle_settings::load(state).await.ok(),
|
|
|
|
|
None => None,
|
|
|
|
|
};
|
|
|
|
|
// No readable row means nothing is known to be in use; the lamps stay
|
|
|
|
|
// quiet rather than failing on data the operator has not entered yet.
|
|
|
|
|
let enabled = settings
|
|
|
|
|
.as_ref()
|
|
|
|
|
.map(|settings| settings.providers_enabled.as_slice())
|
|
|
|
|
.unwrap_or_default();
|
|
|
|
|
let engine = settings
|
|
|
|
|
.as_ref()
|
|
|
|
|
.and_then(|settings| settings.translation_engine.as_deref());
|
|
|
|
|
|
|
|
|
|
let mut probes = Vec::with_capacity(enabled.len());
|
|
|
|
|
for id in enabled {
|
|
|
|
|
probes.push(probe_provider(state, id).await);
|
|
|
|
|
}
|
|
|
|
|
let translation = match engine {
|
|
|
|
|
Some(engine) => Some(probe_engine(state, engine).await),
|
|
|
|
|
None => None,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
SubtitleHealth {
|
|
|
|
|
providers: probes,
|
|
|
|
|
translation,
|
|
|
|
|
alass: binary_check("alass", state.syncer().binary_path()),
|
|
|
|
|
ffmpeg: binary_check("ffmpeg", state.ffmpeg_binary()),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Reachable and credentials accepted, per enabled provider (#200).
|
|
|
|
|
///
|
|
|
|
|
/// A provider that is enabled but was never attached is unconfigured, not
|
|
|
|
|
/// unreachable: its credentials are bootstrap config that this deployment
|
|
|
|
|
/// simply does not have.
|
|
|
|
|
async fn probe_provider(state: &AppState, id: &str) -> ProviderCheck {
|
|
|
|
|
let check = match state.subtitle_provider(id) {
|
|
|
|
|
Some(provider) => check_from(provider.probe().await),
|
|
|
|
|
None => Check::unconfigured(format!(
|
|
|
|
|
"{id} is enabled but not configured — credentials are bootstrap config"
|
|
|
|
|
)),
|
|
|
|
|
};
|
|
|
|
|
ProviderCheck {
|
|
|
|
|
id: id.to_owned(),
|
|
|
|
|
status: check.status,
|
|
|
|
|
detail: check.detail,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Reachable, credentials accepted — for the remote-command backend,
|
|
|
|
|
/// "reachable" means the command ran and exited cleanly (#200).
|
|
|
|
|
async fn probe_engine(state: &AppState, engine: &str) -> Check {
|
|
|
|
|
match state.translation_backend(engine) {
|
|
|
|
|
Some(backend) => backend_check(backend.probe().await),
|
|
|
|
|
None => Check::unconfigured(format!(
|
|
|
|
|
"'{engine}' is selected but unavailable in this build"
|
|
|
|
|
)),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// One probe verdict, whatever kind of upstream produced it. A refused key
|
|
|
|
|
/// gets its own wording because it never resolves by retrying. The two error
|
|
|
|
|
/// enums — providers' and backends' — carry the same shape for this purpose.
|
|
|
|
|
fn check_from(result: Result<(), arr_subs::Error>) -> Check {
|
|
|
|
|
match result {
|
|
|
|
|
Ok(()) => Check::ok(),
|
|
|
|
|
Err(arr_subs::Error::Unauthorized { .. }) => Check::unreachable("credentials refused"),
|
|
|
|
|
Err(error) => Check::unreachable(error.to_string()),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn backend_check(result: Result<(), backend_error::Error>) -> Check {
|
|
|
|
|
match result {
|
|
|
|
|
Ok(()) => Check::ok(),
|
|
|
|
|
Err(backend_error::Error::Unauthorized { .. }) => {
|
|
|
|
|
Check::unreachable("credentials refused")
|
|
|
|
|
}
|
|
|
|
|
Err(error) => Check::unreachable(error.to_string()),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A binary lamp (#200): present and executable at its configured path.
|
|
|
|
|
/// Nothing is spawned — the endpoint is polled, and starting `ffmpeg` per
|
|
|
|
|
/// poll would be neither cheap nor side-effect-free.
|
|
|
|
|
fn binary_check(name: &str, binary: &OsStr) -> Check {
|
|
|
|
|
if binary_present(binary) {
|
|
|
|
|
Check::ok()
|
|
|
|
|
} else {
|
|
|
|
|
Check::unreachable(format!("{name} not found at {}", binary.to_string_lossy()))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
mod tests {
|
|
|
|
|
use std::sync::Arc;
|
|
|
|
|
|
|
|
|
|
use arr_db::Db;
|
|
|
|
|
use arr_subs::{
|
|
|
|
|
Backend, CandidateId, DownloadFuture, Provider, ProviderId, SearchFuture, SearchRequest,
|
|
|
|
|
Syncer,
|
|
|
|
|
};
|
|
|
|
|
use arr_subs::translate as backend;
|
|
|
|
|
|
|
|
|
|
use crate::{router, AppState, Upstreams};
|
|
|
|
|
|
|
|
|
|
/// A provider whose lamp is what `lamp` says — the probe under test is
|
|
|
|
|
/// ours, so the stub never touches the network.
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
struct StubProvider {
|
|
|
|
|
id: &'static str,
|
|
|
|
|
up: bool,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Provider for StubProvider {
|
|
|
|
|
fn id(&self) -> ProviderId {
|
|
|
|
|
ProviderId::new(self.id)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn search<'a>(&'a self, _request: &'a SearchRequest) -> SearchFuture<'a> {
|
|
|
|
|
Box::pin(async move { Ok(Vec::new()) })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn download<'a>(&'a self, _id: &'a CandidateId) -> DownloadFuture<'a> {
|
|
|
|
|
unreachable!("health probes never download");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn probe(&self) -> arr_subs::ProbeFuture<'_> {
|
|
|
|
|
let result = if self.up {
|
|
|
|
|
Ok(())
|
|
|
|
|
} else {
|
|
|
|
|
Err(arr_subs::Error::Unauthorized {
|
|
|
|
|
provider: ProviderId::new(self.id),
|
|
|
|
|
})
|
|
|
|
|
};
|
|
|
|
|
Box::pin(async move { result })
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Same idea for the selected engine; the closure rebuilds its error per
|
|
|
|
|
/// call because probing borrows.
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
struct StubBackend {
|
|
|
|
|
#[allow(dead_code)]
|
|
|
|
|
detail: &'static str,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Backend for StubBackend {
|
|
|
|
|
fn id(&self) -> backend::BackendId {
|
|
|
|
|
backend::BackendId::new("openai")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn supports(&self, _target: &arr_core::Language) -> bool {
|
|
|
|
|
true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn translate<'a>(&'a self, _batch: &'a backend::Batch) -> backend::TranslateFuture<'a> {
|
|
|
|
|
unreachable!("health probes never translate");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn probe(&self) -> backend::ProbeFuture<'_> {
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
Err(backend::Error::Transport {
|
|
|
|
|
backend: backend::BackendId::new("openai"),
|
|
|
|
|
source: "connection refused".into(),
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Serve the app on an ephemeral port and return its base URL.
|
|
|
|
|
async fn serve(state: AppState) -> String {
|
|
|
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
|
|
|
.await
|
|
|
|
|
.expect("bind");
|
|
|
|
|
let address = listener.local_addr().expect("address");
|
|
|
|
|
tokio::spawn(async move { axum::serve(listener, router(state)).await.expect("serve") });
|
|
|
|
|
format!("http://{address}")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// An app with a migrated database and both binaries somewhere findable.
|
|
|
|
|
/// The three classic upstreams point at a dead port; these tests read the
|
|
|
|
|
/// subtitle lamps, not theirs.
|
|
|
|
|
async fn application() -> (tempfile::TempDir, AppState) {
|
|
|
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
|
|
|
let database = Db::connect(dir.path().join("arr.db"))
|
|
|
|
|
.await
|
|
|
|
|
.expect("connect");
|
|
|
|
|
database.migrate().await.expect("migrate");
|
|
|
|
|
let state = AppState::new(Upstreams::new(
|
|
|
|
|
"http://127.0.0.1:1".into(),
|
|
|
|
|
"http://127.0.0.1:1".into(),
|
|
|
|
|
))
|
|
|
|
|
.expect("state")
|
|
|
|
|
.with_database(database)
|
|
|
|
|
.with_syncer(Syncer::new().with_binary("sh"))
|
|
|
|
|
.with_ffmpeg_binary("sh");
|
|
|
|
|
(dir, state)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async fn report(state: AppState) -> serde_json::Value {
|
|
|
|
|
let base = serve(state).await;
|
|
|
|
|
reqwest::get(format!("{base}/api/health"))
|
|
|
|
|
.await
|
|
|
|
|
.expect("request health")
|
|
|
|
|
.json()
|
|
|
|
|
.await
|
|
|
|
|
.expect("health body")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Replace the seeded enabled set. Runtime-checked rather than going
|
|
|
|
|
/// through the settings API, whose validation refuses engines this test
|
|
|
|
|
/// binary has not compiled in.
|
|
|
|
|
async fn set_enabled(state: &AppState, providers: &str, engine: Option<&str>) {
|
|
|
|
|
sqlx::query("UPDATE subtitle_settings SET providers_enabled = ?, translation_engine = ?")
|
|
|
|
|
.bind(providers)
|
|
|
|
|
.bind(engine)
|
|
|
|
|
.execute(state.database().expect("database").pool())
|
|
|
|
|
.await
|
|
|
|
|
.expect("settings update");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn without_a_settings_row_the_lane_is_quiet_and_binaries_still_checked() {
|
|
|
|
|
// No database attached: nothing is known to be in use, so nothing
|
|
|
|
|
// may fail a lamp — but a missing binary is a fact about the host.
|
|
|
|
|
let state = AppState::new(Upstreams::new(
|
|
|
|
|
"http://127.0.0.1:1".into(),
|
|
|
|
|
"http://127.0.0.1:1".into(),
|
|
|
|
|
))
|
|
|
|
|
.expect("state")
|
|
|
|
|
.with_syncer(Syncer::new().with_binary("sh"))
|
|
|
|
|
.with_ffmpeg_binary("/nowhere/ffmpeg");
|
|
|
|
|
|
|
|
|
|
let body = report(state).await;
|
|
|
|
|
assert_eq!(body["subtitles"]["providers"], serde_json::json!([]));
|
|
|
|
|
assert!(body["subtitles"]["translation"].is_null());
|
|
|
|
|
assert_eq!(body["subtitles"]["alass"]["status"], "ok");
|
|
|
|
|
assert_eq!(body["subtitles"]["ffmpeg"]["status"], "unreachable");
|
|
|
|
|
assert_eq!(body["status"], "degraded");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn an_enabled_provider_without_credentials_is_unconfigured() {
|
|
|
|
|
let (_dir, state) = application().await;
|
|
|
|
|
// The seed row enables opensubtitles and podnapisi; none are attached.
|
|
|
|
|
let body = report(state).await;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
body["subtitles"]["providers"],
|
|
|
|
|
serde_json::json!([
|
|
|
|
|
{ "id": "opensubtitles", "status": "unconfigured",
|
|
|
|
|
"detail": "opensubtitles is enabled but not configured — credentials are bootstrap config" },
|
|
|
|
|
{ "id": "podnapisi", "status": "unconfigured",
|
|
|
|
|
"detail": "podnapisi is enabled but not configured — credentials are bootstrap config" },
|
|
|
|
|
])
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(body["status"], "degraded");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn an_attached_provider_lamp_follows_its_probe() {
|
|
|
|
|
let (_dir, state) = application().await;
|
|
|
|
|
set_enabled(&state, r#"["ok","bad"]"#, None).await;
|
|
|
|
|
let state = state.with_subtitle_providers(vec![
|
|
|
|
|
Arc::new(StubProvider { id: "ok", up: true }),
|
|
|
|
|
Arc::new(StubProvider { id: "bad", up: false }),
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
let body = report(state).await;
|
|
|
|
|
assert_eq!(body["subtitles"]["providers"][0]["id"], "ok");
|
|
|
|
|
assert_eq!(body["subtitles"]["providers"][0]["status"], "ok");
|
|
|
|
|
assert_eq!(body["subtitles"]["providers"][1]["id"], "bad");
|
|
|
|
|
assert_eq!(body["subtitles"]["providers"][1]["status"], "unreachable");
|
|
|
|
|
assert_eq!(body["subtitles"]["providers"][1]["detail"], "credentials refused");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn a_selected_engine_is_probed_only_when_selected() {
|
|
|
|
|
let (_dir, state) = application().await;
|
|
|
|
|
let body = report(state.clone()).await;
|
|
|
|
|
assert!(body["subtitles"]["translation"].is_null(), "{body}");
|
|
|
|
|
|
|
|
|
|
set_enabled(&state, "[]", Some("openai")).await;
|
|
|
|
|
// Selected but never attached: unavailable in this deployment.
|
|
|
|
|
let body = report(state).await;
|
|
|
|
|
assert_eq!(body["subtitles"]["translation"]["status"], "unconfigured");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn an_unreachable_engine_fails_the_whole_report() {
|
|
|
|
|
let (_dir, state) = application().await;
|
|
|
|
|
set_enabled(&state, "[]", Some("openai")).await;
|
|
|
|
|
let state = state
|
|
|
|
|
.with_translation_backends(vec![Arc::new(StubBackend {
|
|
|
|
|
detail: "connection refused",
|
|
|
|
|
})]);
|
|
|
|
|
|
|
|
|
|
let body = report(state).await;
|
|
|
|
|
assert_eq!(body["subtitles"]["translation"]["status"], "unreachable");
|
|
|
|
|
assert_eq!(body["status"], "degraded");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|