feat(arr): OpenAI-compatible translation backend
Behind the existing translate-openai feature. One JSON-in/JSON-out chat/completions request per batch; refusal, content-filter and truncated replies fail loudly instead of validating as a short batch.
This commit is contained in:
Miguel Palhas
@@ -0,0 +1,328 @@
|
||||
//! OpenAI-compatible backend tests (`DESIGN.md` §15, issue #191). Everything
|
||||
//! runs against `wiremock` — the live service is never touched.
|
||||
//!
|
||||
//! Built only with the `translate-openai` feature (`required-features` in
|
||||
//! `Cargo.toml`), same as the module itself: with the feature off, cargo
|
||||
//! skips this target rather than building an empty one.
|
||||
|
||||
// Same per-target quirk as the crate's own tests: an integration test links
|
||||
// the library's dependencies without using them all directly.
|
||||
use {
|
||||
arr_parse as _, chardetng as _, encoding_rs as _, reqwest as _, serde as _, serde_json as _,
|
||||
tempfile as _, thiserror as _, tracing as _, zip as _,
|
||||
};
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use arr_core::Language;
|
||||
use arr_subs::translate::{translate, Backend, Error};
|
||||
use arr_subs::{OpenAi, OpenAiConfig};
|
||||
use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
fn backend(server: &MockServer, api_key: Option<&str>) -> OpenAi {
|
||||
OpenAi::with_base_url(
|
||||
"gpt-4o-mini",
|
||||
OpenAiConfig {
|
||||
api_key: api_key.map(str::to_owned),
|
||||
},
|
||||
&format!("{}/v1/", server.uri()),
|
||||
)
|
||||
.expect("client builds")
|
||||
}
|
||||
|
||||
fn chat_response(body: &str) -> String {
|
||||
format!(r#"{{"choices": [{{"message": {{"content": {body}}}, "finish_reason": "stop"}}]}}"#)
|
||||
}
|
||||
|
||||
async fn mount_reply(server: &MockServer, response: ResponseTemplate) {
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(response)
|
||||
.mount(server)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_batch_translates_and_timings_never_move() {
|
||||
let server = MockServer::start().await;
|
||||
let body = chat_response(
|
||||
r#""[{\"number\":1,\"text\":\"ol\\u00e1\"},{\"number\":2,\"text\":\"mundo\"}]""#,
|
||||
);
|
||||
mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
|
||||
let backend = backend(&server, Some("sk-test"));
|
||||
|
||||
let cues = vec![
|
||||
arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hello".to_owned(),
|
||||
},
|
||||
arr_subs::Cue {
|
||||
start: Duration::from_secs(1),
|
||||
end: Duration::from_secs(2),
|
||||
text: "world".to_owned(),
|
||||
},
|
||||
];
|
||||
|
||||
let translated = translate(
|
||||
&backend,
|
||||
&cues,
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect("the mocked reply is well-formed");
|
||||
|
||||
assert_eq!(translated[0].text, "ol\u{e1}");
|
||||
assert_eq!(translated[1].text, "mundo");
|
||||
assert_eq!(translated[0].start, cues[0].start);
|
||||
assert_eq!(translated[1].end, cues[1].end);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_api_key_travels_as_a_bearer_header() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.and(header("Authorization", "Bearer sk-test"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_body_string(chat_response(r#""[{\"number\":1,\"text\":\"oi\"}]""#)),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let backend = backend(&server, Some("sk-test"));
|
||||
|
||||
let cues = vec![arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}];
|
||||
|
||||
translate(
|
||||
&backend,
|
||||
&cues,
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect("the mock only answers a bearer-authed request");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_fenced_json_reply_still_parses() {
|
||||
let server = MockServer::start().await;
|
||||
let body = chat_response(r#""```json\n[{\"number\":1,\"text\":\"oi\"}]\n```""#);
|
||||
mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let cues = vec![arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}];
|
||||
|
||||
let translated = translate(
|
||||
&backend,
|
||||
&cues,
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect("a markdown fence around the JSON is still accepted");
|
||||
|
||||
assert_eq!(translated[0].text, "oi");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unauthorized_status_is_reported_as_such() {
|
||||
let server = MockServer::start().await;
|
||||
mount_reply(&server, ResponseTemplate::new(401)).await;
|
||||
let backend = backend(&server, Some("sk-bad"));
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect_err("401 means the key was rejected");
|
||||
|
||||
assert!(matches!(error, Error::Unauthorized { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_rate_limit_carries_its_retry_after() {
|
||||
let server = MockServer::start().await;
|
||||
mount_reply(
|
||||
&server,
|
||||
ResponseTemplate::new(429).insert_header("retry-after", "30"),
|
||||
)
|
||||
.await;
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect_err("429 is a rate limit, not a broken backend");
|
||||
|
||||
match error {
|
||||
Error::RateLimited { retry_after, .. } => {
|
||||
assert_eq!(retry_after, Some(Duration::from_secs(30)));
|
||||
}
|
||||
other => panic!("expected RateLimited, got {other:?}"),
|
||||
}
|
||||
assert!(error_is_transient(&error));
|
||||
}
|
||||
|
||||
fn error_is_transient(error: &Error) -> bool {
|
||||
error.is_transient()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_content_filtered_reply_is_an_error_not_a_silent_partial() {
|
||||
let server = MockServer::start().await;
|
||||
let body =
|
||||
r#"{"choices": [{"message": {"content": null}, "finish_reason": "content_filter"}]}"#;
|
||||
mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect_err("a content-filtered reply must not translate to nothing silently");
|
||||
|
||||
assert!(matches!(error, Error::Malformed { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_truncated_reply_is_an_error_not_a_silent_partial() {
|
||||
let server = MockServer::start().await;
|
||||
let body = chat_response(r#""[{\"number\":1,\"text\":\"oi""#);
|
||||
let body = body.replace(r#""finish_reason": "stop""#, r#""finish_reason": "length""#);
|
||||
mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect_err("a length-truncated reply must fail loudly");
|
||||
|
||||
assert!(matches!(error, Error::Malformed { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_refusal_is_an_error_not_a_silent_partial() {
|
||||
let server = MockServer::start().await;
|
||||
let body = r#"{"choices": [{"message": {"content": null, "refusal": "I can't help with that."}, "finish_reason": "stop"}]}"#;
|
||||
mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect_err("a refusal must not be read as an empty translation");
|
||||
|
||||
assert!(matches!(error, Error::Malformed { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_mismatched_reply_is_rejected_by_shared_validation() {
|
||||
// The backend does not validate its own replies (DESIGN.md §15) — this
|
||||
// proves the shared `translate()` step catches a dropped cue even when
|
||||
// the backend itself is this one.
|
||||
let server = MockServer::start().await;
|
||||
let body = chat_response(r#""[{\"number\":1,\"text\":\"oi\"}]""#);
|
||||
mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[
|
||||
arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
},
|
||||
arr_subs::Cue {
|
||||
start: Duration::from_secs(1),
|
||||
end: Duration::from_secs(2),
|
||||
text: "there".to_owned(),
|
||||
},
|
||||
],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortuguesePortugal,
|
||||
)
|
||||
.await
|
||||
.expect_err("one cue back for two sent must be rejected");
|
||||
|
||||
assert!(matches!(error, Error::CueMismatch { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unsupported_target_is_refused_before_any_request() {
|
||||
let server = MockServer::start().await;
|
||||
// No mock mounted: a request would fail wiremock's "no matching mock".
|
||||
let backend = backend(&server, None);
|
||||
|
||||
let error = translate(
|
||||
&backend,
|
||||
&[arr_subs::Cue {
|
||||
start: Duration::from_secs(0),
|
||||
end: Duration::from_secs(1),
|
||||
text: "hi".to_owned(),
|
||||
}],
|
||||
&Language::Other("en".to_owned()),
|
||||
&Language::PortugueseUnverified,
|
||||
)
|
||||
.await
|
||||
.expect_err("an unverified variant names no target to hit");
|
||||
|
||||
assert!(matches!(error, Error::UnsupportedTarget { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_backend_answers_to_openai() {
|
||||
let id = OpenAi::new("gpt-4o-mini", OpenAiConfig { api_key: None })
|
||||
.expect("client builds")
|
||||
.id();
|
||||
assert_eq!(id.as_str(), "openai");
|
||||
}
|
||||
Reference in New Issue
Block a user