feat(arr): decode fetched subtitles to UTF-8

Cargo.lock

@@ -205,6 +205,8 @@ name = "arr-subs"
 version = "0.1.0"
 dependencies = [
  "arr-core",
+ "chardetng",
+ "encoding_rs",
  "thiserror",
  "tokio",
 ]
@@ -376,6 +378,17 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "chardetng"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53"
+dependencies = [
+ "cfg-if",
+ "encoding_rs",
+ "memchr",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.45"
@@ -541,6 +554,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"

Cargo.toml

@@ -48,8 +48,10 @@ quick-xml = { version = "0.37", features = ["serialize"] }
 toml = "0.8"
 
 # Odds and ends
+chardetng = "1"
 chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] }
 clap = { version = "4.5", features = ["derive", "env"] }
+encoding_rs = "0.8"
 thiserror = "2"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }

crates/arr-subs/Cargo.toml

@@ -19,6 +19,8 @@ translate-command = []
 
 [dependencies]
 arr-core.workspace = true
+chardetng.workspace = true
+encoding_rs.workspace = true
 thiserror.workspace = true
 
 [dev-dependencies]

crates/arr-subs/src/decode.rs

@@ -0,0 +1,121 @@
+//! Decode a downloaded subtitle's raw bytes to UTF-8.
+//!
+//! [`Fetched::content`] is deliberately raw: providers serve Latin-1 and
+//! Windows-1252 alongside UTF-8, and guessing wrong at the provider boundary
+//! is worse than handing the bytes on (#184). Everything downstream needs
+//! text, though — translation parses cues, `alass` needs a readable file
+//! (#194), and the sidecar written into the title folder is UTF-8 (#195). So
+//! decoding happens exactly once, here, between download and everything
+//! else.
+//!
+//! [`Fetched::content`]: crate::model::Fetched::content
+
+use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection};
+use encoding_rs::Encoding;
+
+use crate::error::Error;
+use crate::model::{CandidateId, Fetched};
+
+impl Fetched {
+    /// Decode this subtitle's raw bytes to UTF-8.
+    ///
+    /// A UTF-8 BOM is honoured when present. Otherwise the encoding is
+    /// detected (`chardetng`) and the bytes are transcoded.
+    ///
+    /// # Errors
+    ///
+    /// [`Error::Decode`] when the bytes do not decode cleanly under the
+    /// detected encoding — a failed attempt, per DESIGN.md §15, rather than
+    /// mojibake written into the library.
+    pub fn decode(&self) -> crate::Result<String> {
+        decode(&self.id, &self.content)
+    }
+}
+
+fn decode(candidate: &CandidateId, bytes: &[u8]) -> crate::Result<String> {
+    let mut detector = EncodingDetector::new(Iso2022JpDetection::Allow);
+    detector.feed(bytes, true);
+    let guess: &'static Encoding = detector.guess(None, Utf8Detection::Allow);
+
+    // `Encoding::decode` sniffs a BOM (UTF-8, UTF-16LE, UTF-16BE) ahead of
+    // `guess`, so a BOM wins regardless of what chardetng thinks the bytes
+    // are.
+    let (decoded, encoding_used, had_errors) = guess.decode(bytes);
+    if had_errors {
+        return Err(Error::Decode {
+            candidate: candidate.clone(),
+            detail: format!("could not decode as {}", encoding_used.name()),
+        });
+    }
+
+    Ok(decoded.into_owned())
+}
+
+#[cfg(test)]
+mod tests {
+    use arr_core::Language;
+
+    use super::Fetched;
+    use crate::model::{CandidateId, SubtitleFormat};
+    use crate::Error;
+
+    fn fetched(content: Vec<u8>) -> Fetched {
+        Fetched {
+            id: CandidateId::new("1"),
+            language: Language::PortuguesePortugal,
+            format: SubtitleFormat::Srt,
+            content,
+        }
+    }
+
+    #[test]
+    fn a_utf8_bom_is_honoured_and_stripped() {
+        let mut content = b"\xef\xbb\xbf".to_vec();
+        content.extend_from_slice("ol\u{e1}".as_bytes());
+
+        let decoded = fetched(content).decode().expect("valid UTF-8 with a BOM");
+
+        assert_eq!(decoded, "ol\u{e1}");
+    }
+
+    #[test]
+    fn plain_ascii_round_trips() {
+        let decoded = fetched(b"hello".to_vec())
+            .decode()
+            .expect("ASCII is valid UTF-8");
+
+        assert_eq!(decoded, "hello");
+    }
+
+    #[test]
+    fn windows_1252_is_detected_and_transcoded() {
+        // Long enough, and Portuguese enough, for chardetng to have a real
+        // signal rather than guessing between several single-byte encodings
+        // that agree on one or two bytes.
+        let text = "Não se pode viver sem paixão. \
+                     Essa não é uma questão de escolha, é uma condição da existência. \
+                     Amanhã há de ser um dia de sol, quem sabe até com uma pitada de \
+                     ironia à mistura.";
+        let (content, encoding_used, had_errors) = encoding_rs::WINDOWS_1252.encode(text);
+        assert!(!had_errors, "the fixture text must round-trip as Latin-1");
+        assert_eq!(encoding_used, encoding_rs::WINDOWS_1252);
+
+        let decoded = fetched(content.into_owned())
+            .decode()
+            .expect("windows-1252 decodes cleanly");
+
+        assert_eq!(decoded, text);
+    }
+
+    #[test]
+    fn bytes_that_cannot_decode_fail_loudly() {
+        // A byte sequence that is invalid under UTF-16, the encoding a
+        // UTF-16LE BOM commits the decoder to.
+        let mut content = b"\xff\xfe".to_vec();
+        content.extend_from_slice(&[0x00, 0xd8]); // an unpaired UTF-16 surrogate
+
+        let error = fetched(content).decode().expect_err("malformed UTF-16");
+
+        assert!(matches!(error, Error::Decode { .. }), "{error:?}");
+    }
+}

crates/arr-subs/src/lib.rs

@@ -20,6 +20,7 @@
 
 use std::{fmt, future::Future, pin::Pin};
 
+pub mod decode;
 pub mod error;
 pub mod model;
 pub mod srt;

crates/arr-subs/src/model.rs

@@ -234,7 +234,7 @@ pub struct Fetched {
     pub format: SubtitleFormat,
     /// The subtitle itself, exactly as it arrived. Not decoded to UTF-8 here:
     /// providers serve Latin-1 and Windows-1252 files, and guessing wrong is
-    /// worse than handing the bytes on.
+    /// worse than handing the bytes on. Call [`Fetched::decode`] to get text.
     pub content: Vec<u8>,
 }