//! Decode a downloaded subtitle's raw bytes to UTF-8. //! //! [`Fetched::content`] is deliberately raw: providers serve Latin-1 and //! Windows-1252 alongside UTF-8, and guessing wrong at the provider boundary //! is worse than handing the bytes on (#184). Everything downstream needs //! text, though — translation parses cues, `alass` needs a readable file //! (#194), and the sidecar written into the title folder is UTF-8 (#195). So //! decoding happens exactly once, here, between download and everything //! else. //! //! [`Fetched::content`]: crate::model::Fetched::content use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection}; use encoding_rs::Encoding; use crate::error::Error; use crate::model::{CandidateId, Fetched}; impl Fetched { /// Decode this subtitle's raw bytes to UTF-8. /// /// A UTF-8 BOM is honoured when present. Otherwise the encoding is /// detected (`chardetng`) and the bytes are transcoded. /// /// # Errors /// /// [`Error::Decode`] when the bytes do not decode cleanly under the /// detected encoding — a failed attempt, per DESIGN.md §15, rather than /// mojibake written into the library. pub fn decode(&self) -> crate::Result { decode(&self.id, &self.content) } } fn decode(candidate: &CandidateId, bytes: &[u8]) -> crate::Result { let mut detector = EncodingDetector::new(Iso2022JpDetection::Allow); detector.feed(bytes, true); let guess: &'static Encoding = detector.guess(None, Utf8Detection::Allow); // `Encoding::decode` sniffs a BOM (UTF-8, UTF-16LE, UTF-16BE) ahead of // `guess`, so a BOM wins regardless of what chardetng thinks the bytes // are. let (decoded, encoding_used, had_errors) = guess.decode(bytes); if had_errors { return Err(Error::Decode { candidate: candidate.clone(), detail: format!("could not decode as {}", encoding_used.name()), }); } Ok(decoded.into_owned()) } #[cfg(test)] mod tests { use arr_core::Language; use super::Fetched; use crate::model::{CandidateId, SubtitleFormat}; use crate::Error; fn fetched(content: Vec) -> Fetched { Fetched { id: CandidateId::new("1"), language: Language::PortuguesePortugal, format: SubtitleFormat::Srt, content, } } #[test] fn a_utf8_bom_is_honoured_and_stripped() { let mut content = b"\xef\xbb\xbf".to_vec(); content.extend_from_slice("ol\u{e1}".as_bytes()); let decoded = fetched(content).decode().expect("valid UTF-8 with a BOM"); assert_eq!(decoded, "ol\u{e1}"); } #[test] fn plain_ascii_round_trips() { let decoded = fetched(b"hello".to_vec()) .decode() .expect("ASCII is valid UTF-8"); assert_eq!(decoded, "hello"); } #[test] fn windows_1252_is_detected_and_transcoded() { // Long enough, and Portuguese enough, for chardetng to have a real // signal rather than guessing between several single-byte encodings // that agree on one or two bytes. let text = "Não se pode viver sem paixão. \ Essa não é uma questão de escolha, é uma condição da existência. \ Amanhã há de ser um dia de sol, quem sabe até com uma pitada de \ ironia à mistura."; let (content, encoding_used, had_errors) = encoding_rs::WINDOWS_1252.encode(text); assert!(!had_errors, "the fixture text must round-trip as Latin-1"); assert_eq!(encoding_used, encoding_rs::WINDOWS_1252); let decoded = fetched(content.into_owned()) .decode() .expect("windows-1252 decodes cleanly"); assert_eq!(decoded, text); } #[test] fn bytes_that_cannot_decode_fail_loudly() { // A byte sequence that is invalid under UTF-16, the encoding a // UTF-16LE BOM commits the decoder to. let mut content = b"\xff\xfe".to_vec(); content.extend_from_slice(&[0x00, 0xd8]); // an unpaired UTF-16 surrogate let error = fetched(content).decode().expect_err("malformed UTF-16"); assert!(matches!(error, Error::Decode { .. }), "{error:?}"); } }