From 81dd60641497ad5db47cfa06dbe889975654d8e0 Mon Sep 17 00:00:00 2001 From: Miguel Palhas Date: Mon, 24 Aug 2026 22:41:17 +0100 Subject: [PATCH] feat(arr): decode fetched subtitles to UTF-8 --- Cargo.lock | 22 +++++++ Cargo.toml | 2 + crates/arr-subs/Cargo.toml | 2 + crates/arr-subs/src/decode.rs | 121 ++++++++++++++++++++++++++++++++++ crates/arr-subs/src/lib.rs | 1 + crates/arr-subs/src/model.rs | 2 +- 6 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 crates/arr-subs/src/decode.rs diff --git a/Cargo.lock b/Cargo.lock index 4ea3fdd..f274207 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,6 +205,8 @@ name = "arr-subs" version = "0.1.0" dependencies = [ "arr-core", + "chardetng", + "encoding_rs", "thiserror", "tokio", ] @@ -376,6 +378,17 @@ dependencies = [ "rand_core", ] +[[package]] +name = "chardetng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.45" @@ -541,6 +554,15 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 5dff684..490fac9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,8 +48,10 @@ quick-xml = { version = "0.37", features = ["serialize"] } toml = "0.8" # Odds and ends +chardetng = "1" chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } clap = { version = "4.5", features = ["derive", "env"] } +encoding_rs = "0.8" thiserror = "2" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/crates/arr-subs/Cargo.toml b/crates/arr-subs/Cargo.toml index bcabaff..155e0f0 100644 --- a/crates/arr-subs/Cargo.toml +++ b/crates/arr-subs/Cargo.toml @@ -19,6 +19,8 @@ translate-command = [] [dependencies] arr-core.workspace = true +chardetng.workspace = true +encoding_rs.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/crates/arr-subs/src/decode.rs b/crates/arr-subs/src/decode.rs new file mode 100644 index 0000000..2dd4102 --- /dev/null +++ b/crates/arr-subs/src/decode.rs @@ -0,0 +1,121 @@ +//! Decode a downloaded subtitle's raw bytes to UTF-8. +//! +//! [`Fetched::content`] is deliberately raw: providers serve Latin-1 and +//! Windows-1252 alongside UTF-8, and guessing wrong at the provider boundary +//! is worse than handing the bytes on (#184). Everything downstream needs +//! text, though — translation parses cues, `alass` needs a readable file +//! (#194), and the sidecar written into the title folder is UTF-8 (#195). So +//! decoding happens exactly once, here, between download and everything +//! else. +//! +//! [`Fetched::content`]: crate::model::Fetched::content + +use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection}; +use encoding_rs::Encoding; + +use crate::error::Error; +use crate::model::{CandidateId, Fetched}; + +impl Fetched { + /// Decode this subtitle's raw bytes to UTF-8. + /// + /// A UTF-8 BOM is honoured when present. Otherwise the encoding is + /// detected (`chardetng`) and the bytes are transcoded. + /// + /// # Errors + /// + /// [`Error::Decode`] when the bytes do not decode cleanly under the + /// detected encoding — a failed attempt, per DESIGN.md §15, rather than + /// mojibake written into the library. + pub fn decode(&self) -> crate::Result { + decode(&self.id, &self.content) + } +} + +fn decode(candidate: &CandidateId, bytes: &[u8]) -> crate::Result { + let mut detector = EncodingDetector::new(Iso2022JpDetection::Allow); + detector.feed(bytes, true); + let guess: &'static Encoding = detector.guess(None, Utf8Detection::Allow); + + // `Encoding::decode` sniffs a BOM (UTF-8, UTF-16LE, UTF-16BE) ahead of + // `guess`, so a BOM wins regardless of what chardetng thinks the bytes + // are. + let (decoded, encoding_used, had_errors) = guess.decode(bytes); + if had_errors { + return Err(Error::Decode { + candidate: candidate.clone(), + detail: format!("could not decode as {}", encoding_used.name()), + }); + } + + Ok(decoded.into_owned()) +} + +#[cfg(test)] +mod tests { + use arr_core::Language; + + use super::Fetched; + use crate::model::{CandidateId, SubtitleFormat}; + use crate::Error; + + fn fetched(content: Vec) -> Fetched { + Fetched { + id: CandidateId::new("1"), + language: Language::PortuguesePortugal, + format: SubtitleFormat::Srt, + content, + } + } + + #[test] + fn a_utf8_bom_is_honoured_and_stripped() { + let mut content = b"\xef\xbb\xbf".to_vec(); + content.extend_from_slice("ol\u{e1}".as_bytes()); + + let decoded = fetched(content).decode().expect("valid UTF-8 with a BOM"); + + assert_eq!(decoded, "ol\u{e1}"); + } + + #[test] + fn plain_ascii_round_trips() { + let decoded = fetched(b"hello".to_vec()) + .decode() + .expect("ASCII is valid UTF-8"); + + assert_eq!(decoded, "hello"); + } + + #[test] + fn windows_1252_is_detected_and_transcoded() { + // Long enough, and Portuguese enough, for chardetng to have a real + // signal rather than guessing between several single-byte encodings + // that agree on one or two bytes. + let text = "Não se pode viver sem paixão. \ + Essa não é uma questão de escolha, é uma condição da existência. \ + Amanhã há de ser um dia de sol, quem sabe até com uma pitada de \ + ironia à mistura."; + let (content, encoding_used, had_errors) = encoding_rs::WINDOWS_1252.encode(text); + assert!(!had_errors, "the fixture text must round-trip as Latin-1"); + assert_eq!(encoding_used, encoding_rs::WINDOWS_1252); + + let decoded = fetched(content.into_owned()) + .decode() + .expect("windows-1252 decodes cleanly"); + + assert_eq!(decoded, text); + } + + #[test] + fn bytes_that_cannot_decode_fail_loudly() { + // A byte sequence that is invalid under UTF-16, the encoding a + // UTF-16LE BOM commits the decoder to. + let mut content = b"\xff\xfe".to_vec(); + content.extend_from_slice(&[0x00, 0xd8]); // an unpaired UTF-16 surrogate + + let error = fetched(content).decode().expect_err("malformed UTF-16"); + + assert!(matches!(error, Error::Decode { .. }), "{error:?}"); + } +} diff --git a/crates/arr-subs/src/lib.rs b/crates/arr-subs/src/lib.rs index 1b6ade9..abbcb83 100644 --- a/crates/arr-subs/src/lib.rs +++ b/crates/arr-subs/src/lib.rs @@ -20,6 +20,7 @@ use std::{fmt, future::Future, pin::Pin}; +pub mod decode; pub mod error; pub mod model; pub mod srt; diff --git a/crates/arr-subs/src/model.rs b/crates/arr-subs/src/model.rs index 33c6e01..bbf976f 100644 --- a/crates/arr-subs/src/model.rs +++ b/crates/arr-subs/src/model.rs @@ -234,7 +234,7 @@ pub struct Fetched { pub format: SubtitleFormat, /// The subtitle itself, exactly as it arrived. Not decoded to UTF-8 here: /// providers serve Latin-1 and Windows-1252 files, and guessing wrong is - /// worse than handing the bytes on. + /// worse than handing the bytes on. Call [`Fetched::decode`] to get text. pub content: Vec, }