arr/crates/arr-subs/tests/openai.rs

//! OpenAI-compatible backend tests (`DESIGN.md` §15, issue #191). Everything
//! runs against `wiremock` — the live service is never touched.
//!
//! Built only with the `translate-openai` feature (`required-features` in
//! `Cargo.toml`), same as the module itself: with the feature off, cargo
//! skips this target rather than building an empty one.

// Same per-target quirk as the crate's own tests: an integration test links
// the library's dependencies without using them all directly.
use {
    arr_parse as _, chardetng as _, encoding_rs as _, reqwest as _, serde as _, serde_json as _,
    tempfile as _, thiserror as _, tracing as _, zip as _,
};

use std::time::Duration;

use arr_core::Language;
use arr_subs::translate::{translate, Backend, Error};
use arr_subs::{OpenAi, OpenAiConfig, OpenAiEndpoint};
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

fn backend(server: &MockServer, api_key: Option<&str>) -> OpenAi {
    OpenAi::with_base_url(
        "gpt-4o-mini",
        OpenAiConfig {
            api_key: api_key.map(str::to_owned),
        },
        &format!("{}/v1/", server.uri()),
    )
    .expect("client builds")
}

fn chat_response(body: &str) -> String {
    format!(r#"{{"choices": [{{"message": {{"content": {body}}}, "finish_reason": "stop"}}]}}"#)
}

async fn mount_reply(server: &MockServer, response: ResponseTemplate) {
    Mock::given(method("POST"))
        .and(path("/v1/chat/completions"))
        .respond_with(response)
        .mount(server)
        .await;
}

#[tokio::test]
async fn a_batch_translates_and_timings_never_move() {
    let server = MockServer::start().await;
    let body = chat_response(
        r#""[{\"number\":1,\"text\":\"ol\\u00e1\"},{\"number\":2,\"text\":\"mundo\"}]""#,
    );
    mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
    let backend = backend(&server, Some("sk-test"));

    let cues = vec![
        arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hello".to_owned(),
        },
        arr_subs::Cue {
            start: Duration::from_secs(1),
            end: Duration::from_secs(2),
            text: "world".to_owned(),
        },
    ];

    let translated = translate(
        &backend,
        &cues,
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect("the mocked reply is well-formed");

    assert_eq!(translated[0].text, "ol\u{e1}");
    assert_eq!(translated[1].text, "mundo");
    assert_eq!(translated[0].start, cues[0].start);
    assert_eq!(translated[1].end, cues[1].end);
}

#[tokio::test]
async fn the_api_key_travels_as_a_bearer_header() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/chat/completions"))
        .and(header("Authorization", "Bearer sk-test"))
        .respond_with(
            ResponseTemplate::new(200)
                .set_body_string(chat_response(r#""[{\"number\":1,\"text\":\"oi\"}]""#)),
        )
        .mount(&server)
        .await;
    let backend = backend(&server, Some("sk-test"));

    let cues = vec![arr_subs::Cue {
        start: Duration::from_secs(0),
        end: Duration::from_secs(1),
        text: "hi".to_owned(),
    }];

    translate(
        &backend,
        &cues,
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect("the mock only answers a bearer-authed request");
}

#[tokio::test]
async fn a_fenced_json_reply_still_parses() {
    let server = MockServer::start().await;
    let body = chat_response(r#""```json\n[{\"number\":1,\"text\":\"oi\"}]\n```""#);
    mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
    let backend = backend(&server, None);

    let cues = vec![arr_subs::Cue {
        start: Duration::from_secs(0),
        end: Duration::from_secs(1),
        text: "hi".to_owned(),
    }];

    let translated = translate(
        &backend,
        &cues,
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect("a markdown fence around the JSON is still accepted");

    assert_eq!(translated[0].text, "oi");
}

#[tokio::test]
async fn unauthorized_status_is_reported_as_such() {
    let server = MockServer::start().await;
    mount_reply(&server, ResponseTemplate::new(401)).await;
    let backend = backend(&server, Some("sk-bad"));

    let error = translate(
        &backend,
        &[arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hi".to_owned(),
        }],
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect_err("401 means the key was rejected");

    assert!(matches!(error, Error::Unauthorized { .. }));
}

#[tokio::test]
async fn a_rate_limit_carries_its_retry_after() {
    let server = MockServer::start().await;
    mount_reply(
        &server,
        ResponseTemplate::new(429).insert_header("retry-after", "30"),
    )
    .await;
    let backend = backend(&server, None);

    let error = translate(
        &backend,
        &[arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hi".to_owned(),
        }],
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect_err("429 is a rate limit, not a broken backend");

    match error {
        Error::RateLimited { retry_after, .. } => {
            assert_eq!(retry_after, Some(Duration::from_secs(30)));
        }
        other => panic!("expected RateLimited, got {other:?}"),
    }
    assert!(error_is_transient(&error));
}

fn error_is_transient(error: &Error) -> bool {
    error.is_transient()
}

#[tokio::test]
async fn a_content_filtered_reply_is_an_error_not_a_silent_partial() {
    let server = MockServer::start().await;
    let body =
        r#"{"choices": [{"message": {"content": null}, "finish_reason": "content_filter"}]}"#;
    mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
    let backend = backend(&server, None);

    let error = translate(
        &backend,
        &[arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hi".to_owned(),
        }],
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect_err("a content-filtered reply must not translate to nothing silently");

    assert!(matches!(error, Error::Malformed { .. }));
}

#[tokio::test]
async fn a_truncated_reply_is_an_error_not_a_silent_partial() {
    let server = MockServer::start().await;
    let body = chat_response(r#""[{\"number\":1,\"text\":\"oi""#);
    let body = body.replace(r#""finish_reason": "stop""#, r#""finish_reason": "length""#);
    mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
    let backend = backend(&server, None);

    let error = translate(
        &backend,
        &[arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hi".to_owned(),
        }],
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect_err("a length-truncated reply must fail loudly");

    assert!(matches!(error, Error::Malformed { .. }));
}

#[tokio::test]
async fn a_refusal_is_an_error_not_a_silent_partial() {
    let server = MockServer::start().await;
    let body = r#"{"choices": [{"message": {"content": null, "refusal": "I can't help with that."}, "finish_reason": "stop"}]}"#;
    mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
    let backend = backend(&server, None);

    let error = translate(
        &backend,
        &[arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hi".to_owned(),
        }],
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect_err("a refusal must not be read as an empty translation");

    assert!(matches!(error, Error::Malformed { .. }));
}

#[tokio::test]
async fn a_mismatched_reply_is_rejected_by_shared_validation() {
    // The backend does not validate its own replies (DESIGN.md §15) — this
    // proves the shared `translate()` step catches a dropped cue even when
    // the backend itself is this one.
    let server = MockServer::start().await;
    let body = chat_response(r#""[{\"number\":1,\"text\":\"oi\"}]""#);
    mount_reply(&server, ResponseTemplate::new(200).set_body_string(body)).await;
    let backend = backend(&server, None);

    let error = translate(
        &backend,
        &[
            arr_subs::Cue {
                start: Duration::from_secs(0),
                end: Duration::from_secs(1),
                text: "hi".to_owned(),
            },
            arr_subs::Cue {
                start: Duration::from_secs(1),
                end: Duration::from_secs(2),
                text: "there".to_owned(),
            },
        ],
        &Language::Other("en".to_owned()),
        &Language::PortuguesePortugal,
    )
    .await
    .expect_err("one cue back for two sent must be rejected");

    assert!(matches!(error, Error::CueMismatch { .. }));
}

#[tokio::test]
async fn an_unsupported_target_is_refused_before_any_request() {
    let server = MockServer::start().await;
    // No mock mounted: a request would fail wiremock's "no matching mock".
    let backend = backend(&server, None);

    let error = translate(
        &backend,
        &[arr_subs::Cue {
            start: Duration::from_secs(0),
            end: Duration::from_secs(1),
            text: "hi".to_owned(),
        }],
        &Language::Other("en".to_owned()),
        &Language::PortugueseUnverified,
    )
    .await
    .expect_err("an unverified variant names no target to hit");

    assert!(matches!(error, Error::UnsupportedTarget { .. }));
}

/// #220: an edit of `openai_base_url` / `openai_model` reaches the running
/// backend without a restart — the next batch goes to the new endpoint and
/// names the new model, with no reconstruction in between.
#[tokio::test]
async fn repointing_the_endpoint_moves_the_next_batch() {
    let first = MockServer::start().await;
    let second = MockServer::start().await;
    mount_reply(
        &first,
        ResponseTemplate::new(200).set_body_string(chat_response(
            r#""[{\"number\": 1, \"text\": \"do primeiro\"}]""#,
        )),
    )
    .await;
    mount_reply(
        &second,
        ResponseTemplate::new(200).set_body_string(chat_response(
            r#""[{\"number\": 1, \"text\": \"do segundo\"}]""#,
        )),
    )
    .await;

    let endpoint = OpenAiEndpoint::new(Some(&format!("{}/v1/", first.uri())), Some("first-model"))
        .expect("endpoint resolves");
    let backend =
        OpenAi::with_endpoint(OpenAiConfig { api_key: None }, endpoint.clone()).expect("builds");

    let cues = vec![arr_subs::translate::BatchCue {
        number: 1,
        text: "from the first".to_owned(),
    }];
    let batch = arr_subs::translate::Batch {
        source: Language::Other("en".to_owned()),
        target: Language::PortuguesePortugal,
        cues: cues.clone(),
    };
    let out = backend.translate(&batch).await.expect("first batch");
    assert_eq!(out[0].text, "do primeiro");

    endpoint
        .set(Some(&format!("{}/v1/", second.uri())), Some("second-model"))
        .expect("repoint");
    let out = backend.translate(&batch).await.expect("second batch");
    assert_eq!(out[0].text, "do segundo");

    let sent = second.received_requests().await.expect("requests recorded");
    let body: serde_json::Value =
        serde_json::from_slice(&sent.last().expect("one request").body).expect("json body");
    assert_eq!(body["model"], "second-model");
}

#[test]
fn the_backend_answers_to_openai() {
    let id = OpenAi::new(OpenAiConfig { api_key: None })
        .expect("client builds")
        .id();
    assert_eq!(id.as_str(), "openai");
}