122 lines
4.2 KiB
Rust
122 lines
4.2 KiB
Rust
//! Decode a downloaded subtitle's raw bytes to UTF-8.
|
|
//!
|
|
//! [`Fetched::content`] is deliberately raw: providers serve Latin-1 and
|
|
//! Windows-1252 alongside UTF-8, and guessing wrong at the provider boundary
|
|
//! is worse than handing the bytes on (#184). Everything downstream needs
|
|
//! text, though — translation parses cues, `alass` needs a readable file
|
|
//! (#194), and the sidecar written into the title folder is UTF-8 (#195). So
|
|
//! decoding happens exactly once, here, between download and everything
|
|
//! else.
|
|
//!
|
|
//! [`Fetched::content`]: crate::model::Fetched::content
|
|
|
|
use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection};
|
|
use encoding_rs::Encoding;
|
|
|
|
use crate::error::Error;
|
|
use crate::model::{CandidateId, Fetched};
|
|
|
|
impl Fetched {
|
|
/// Decode this subtitle's raw bytes to UTF-8.
|
|
///
|
|
/// A UTF-8 BOM is honoured when present. Otherwise the encoding is
|
|
/// detected (`chardetng`) and the bytes are transcoded.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// [`Error::Decode`] when the bytes do not decode cleanly under the
|
|
/// detected encoding — a failed attempt, per DESIGN.md §15, rather than
|
|
/// mojibake written into the library.
|
|
pub fn decode(&self) -> crate::Result<String> {
|
|
decode(&self.id, &self.content)
|
|
}
|
|
}
|
|
|
|
fn decode(candidate: &CandidateId, bytes: &[u8]) -> crate::Result<String> {
|
|
let mut detector = EncodingDetector::new(Iso2022JpDetection::Allow);
|
|
detector.feed(bytes, true);
|
|
let guess: &'static Encoding = detector.guess(None, Utf8Detection::Allow);
|
|
|
|
// `Encoding::decode` sniffs a BOM (UTF-8, UTF-16LE, UTF-16BE) ahead of
|
|
// `guess`, so a BOM wins regardless of what chardetng thinks the bytes
|
|
// are.
|
|
let (decoded, encoding_used, had_errors) = guess.decode(bytes);
|
|
if had_errors {
|
|
return Err(Error::Decode {
|
|
candidate: candidate.clone(),
|
|
detail: format!("could not decode as {}", encoding_used.name()),
|
|
});
|
|
}
|
|
|
|
Ok(decoded.into_owned())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use arr_core::Language;
|
|
|
|
use super::Fetched;
|
|
use crate::model::{CandidateId, SubtitleFormat};
|
|
use crate::Error;
|
|
|
|
fn fetched(content: Vec<u8>) -> Fetched {
|
|
Fetched {
|
|
id: CandidateId::new("1"),
|
|
language: Language::PortuguesePortugal,
|
|
format: SubtitleFormat::Srt,
|
|
content,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_utf8_bom_is_honoured_and_stripped() {
|
|
let mut content = b"\xef\xbb\xbf".to_vec();
|
|
content.extend_from_slice("ol\u{e1}".as_bytes());
|
|
|
|
let decoded = fetched(content).decode().expect("valid UTF-8 with a BOM");
|
|
|
|
assert_eq!(decoded, "ol\u{e1}");
|
|
}
|
|
|
|
#[test]
|
|
fn plain_ascii_round_trips() {
|
|
let decoded = fetched(b"hello".to_vec())
|
|
.decode()
|
|
.expect("ASCII is valid UTF-8");
|
|
|
|
assert_eq!(decoded, "hello");
|
|
}
|
|
|
|
#[test]
|
|
fn windows_1252_is_detected_and_transcoded() {
|
|
// Long enough, and Portuguese enough, for chardetng to have a real
|
|
// signal rather than guessing between several single-byte encodings
|
|
// that agree on one or two bytes.
|
|
let text = "Não se pode viver sem paixão. \
|
|
Essa não é uma questão de escolha, é uma condição da existência. \
|
|
Amanhã há de ser um dia de sol, quem sabe até com uma pitada de \
|
|
ironia à mistura.";
|
|
let (content, encoding_used, had_errors) = encoding_rs::WINDOWS_1252.encode(text);
|
|
assert!(!had_errors, "the fixture text must round-trip as Latin-1");
|
|
assert_eq!(encoding_used, encoding_rs::WINDOWS_1252);
|
|
|
|
let decoded = fetched(content.into_owned())
|
|
.decode()
|
|
.expect("windows-1252 decodes cleanly");
|
|
|
|
assert_eq!(decoded, text);
|
|
}
|
|
|
|
#[test]
|
|
fn bytes_that_cannot_decode_fail_loudly() {
|
|
// A byte sequence that is invalid under UTF-16, the encoding a
|
|
// UTF-16LE BOM commits the decoder to.
|
|
let mut content = b"\xff\xfe".to_vec();
|
|
content.extend_from_slice(&[0x00, 0xd8]); // an unpaired UTF-16 surrogate
|
|
|
|
let error = fetched(content).decode().expect_err("malformed UTF-16");
|
|
|
|
assert!(matches!(error, Error::Decode { .. }), "{error:?}");
|
|
}
|
|
}
|