Files
arr/crates/arr-subs/src/openai.rs
T

386 lines
12 KiB
Rust

//! The OpenAI-compatible translation backend (`DESIGN.md` §15, issue #191).
//!
//! One HTTP shape — the `/chat/completions` endpoint `OpenAI` defined — reaches
//! the largest number of options: `OpenAI` itself, `OpenRouter`, Groq, a
//! self-hosted gateway, or a local `llama.cpp` server. The model and base URL
//! are constructor arguments rather than fields on [`OpenAiConfig`]: which
//! engine runs, and where, is a database setting (DESIGN.md §15, issue #198).
//! Only the credential is bootstrap config or environment, per §10 — and even
//! that is optional, since a self-hosted gateway may not ask for one.
//!
//! The batch travels as a JSON array of `{number, text}` objects, and the
//! reply is asked for in the same shape: JSON round-trips a cue's embedded
//! newlines cleanly, where a line-oriented reply format could not tell a
//! multi-line cue from two cues. [`translate::translate`] chunks, validates
//! and reassembles; this module only builds one request and turns one reply
//! into [`TranslatedCue`]s. A refusal, a content-filtered reply or a reply cut
//! short by the token limit each fail with their own [`Error::Malformed`]
//! detail rather than reaching validation as a silently short batch.
use std::fmt;
use std::time::Duration;
use reqwest::{Client, StatusCode, Url};
use serde::{Deserialize, Serialize};
use arr_core::Language;
use crate::translate::{
strip_code_fence, system_prompt, Backend, BackendId, Batch, Error, Result, TranslateFuture,
TranslatedCue,
};
const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1/";
const BACKEND_NAME: &str = "openai";
const USER_AGENT: &str = "arr v0.1.0";
const MAX_ERROR_BODY: usize = 512;
/// Credentials for an OpenAI-compatible endpoint (DESIGN.md §10).
///
/// Never persisted to the database — it reaches here from bootstrap config or
/// environment only. Optional because a self-hosted gateway (a local
/// `llama.cpp` server, say) may not require one.
#[derive(Clone, PartialEq, Eq)]
pub struct OpenAiConfig {
/// Sent as an `Authorization: Bearer` header when present.
pub api_key: Option<String>,
}
// Hand-written: a derived Debug would print the key.
impl fmt::Debug for OpenAiConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OpenAiConfig")
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
.finish()
}
}
/// The OpenAI-compatible backend behind the [`Backend`] trait.
pub struct OpenAi {
http: Client,
base_url: Url,
config: OpenAiConfig,
model: String,
id: BackendId,
}
// Hand-written: the config must never reach a log line.
impl fmt::Debug for OpenAi {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OpenAi")
.field("base_url", &self.base_url.as_str())
.field("model", &self.model)
.finish_non_exhaustive()
}
}
impl OpenAi {
/// A client against `https://api.openai.com/v1/`.
///
/// # Errors
///
/// Fails if the HTTP client cannot be constructed.
pub fn new(model: impl Into<String>, config: OpenAiConfig) -> Result<Self> {
Self::with_base_url(model, config, DEFAULT_BASE_URL)
}
/// Point the client at another OpenAI-compatible endpoint — `OpenRouter`,
/// Groq, a self-hosted gateway, a local `llama.cpp` server, or `wiremock`
/// in tests.
///
/// # Errors
///
/// Same as [`Self::new`], plus a `base_url` that does not parse.
pub fn with_base_url(
model: impl Into<String>,
config: OpenAiConfig,
base_url: &str,
) -> Result<Self> {
let id = BackendId::new(BACKEND_NAME);
let http = Client::builder().build().map_err(|err| Error::Transport {
backend: id.clone(),
source: Box::new(err),
})?;
let base_url = base_url.parse().map_err(|err| Error::Malformed {
backend: id.clone(),
detail: format!("bad base URL: {err}"),
})?;
Ok(Self {
http,
base_url,
config,
model: model.into(),
id,
})
}
async fn check_status(&self, response: reqwest::Response) -> Result<reqwest::Response> {
match response.status() {
StatusCode::OK => Ok(response),
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(Error::Unauthorized {
backend: self.id.clone(),
}),
StatusCode::TOO_MANY_REQUESTS => Err(Error::RateLimited {
backend: self.id.clone(),
retry_after: retry_after(&response),
}),
other => {
let body = response.text().await.unwrap_or_default();
Err(Error::Malformed {
backend: self.id.clone(),
detail: format!("status {}: {}", other.as_u16(), truncate(&body)),
})
}
}
}
async fn decode<T: for<'de> Deserialize<'de>>(&self, response: reqwest::Response) -> Result<T> {
let bytes = response.bytes().await.map_err(|err| Error::Transport {
backend: self.id.clone(),
source: Box::new(err),
})?;
serde_json::from_slice(&bytes).map_err(|err| Error::Malformed {
backend: self.id.clone(),
detail: err.to_string(),
})
}
async fn translate_inner(&self, batch: &Batch) -> Result<Vec<TranslatedCue>> {
let request_cues: Vec<RequestCue<'_>> = batch
.cues
.iter()
.map(|cue| RequestCue {
number: cue.number,
text: &cue.text,
})
.collect();
let user_content =
serde_json::to_string(&request_cues).map_err(|err| Error::Malformed {
backend: self.id.clone(),
detail: format!("could not encode batch: {err}"),
})?;
let body = ChatRequest {
model: &self.model,
messages: vec![
ChatMessage {
role: "system",
content: system_prompt(&batch.source, &batch.target),
},
ChatMessage {
role: "user",
content: user_content,
},
],
};
let url = self
.base_url
.join("chat/completions")
.map_err(|err| Error::Malformed {
backend: self.id.clone(),
detail: format!("bad chat completions path: {err}"),
})?;
let mut request = self
.http
.post(url)
.header("User-Agent", USER_AGENT)
.json(&body);
if let Some(api_key) = self.config.api_key.as_deref() {
request = request.bearer_auth(api_key);
}
tracing::debug!(backend = %self.id, model = %self.model, "OpenAI translate request");
let response = request.send().await.map_err(|err| Error::Transport {
backend: self.id.clone(),
source: Box::new(err),
})?;
let response = self.check_status(response).await?;
let reply: ChatResponse = self.decode(response).await?;
self.parse_reply(reply)
}
fn parse_reply(&self, reply: ChatResponse) -> Result<Vec<TranslatedCue>> {
let malformed = |detail: String| Error::Malformed {
backend: self.id.clone(),
detail,
};
let choice = reply
.choices
.into_iter()
.next()
.ok_or_else(|| malformed("no choices in reply".to_owned()))?;
if let Some(refusal) = choice.message.refusal.filter(|refusal| !refusal.is_empty()) {
return Err(malformed(format!("backend refused: {refusal}")));
}
match choice.finish_reason.as_deref() {
Some("content_filter") => {
return Err(malformed("reply blocked by content filter".to_owned()))
}
Some("length") => {
return Err(malformed(
"reply was truncated before it finished".to_owned(),
))
}
_ => {}
}
let content = choice
.message
.content
.filter(|content| !content.trim().is_empty())
.ok_or_else(|| malformed("empty reply".to_owned()))?;
let cues: Vec<ReplyCue> = serde_json::from_str(strip_code_fence(&content))
.map_err(|err| malformed(format!("reply was not the expected JSON: {err}")))?;
Ok(cues
.into_iter()
.map(|cue| TranslatedCue {
number: cue.number,
text: cue.text,
})
.collect())
}
}
impl Backend for OpenAi {
fn id(&self) -> BackendId {
self.id.clone()
}
/// True for every target except [`Language::PortugueseUnverified`]: an
/// unverified variant names no exact target to translate into, and an LLM
/// asked to hit one anyway would have to guess (DESIGN.md §15).
fn supports(&self, target: &Language) -> bool {
!matches!(target, Language::PortugueseUnverified)
}
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
Box::pin(async move { self.translate_inner(batch).await })
}
}
fn retry_after(response: &reqwest::Response) -> Option<Duration> {
response
.headers()
.get(reqwest::header::RETRY_AFTER)?
.to_str()
.ok()?
.trim()
.parse::<u64>()
.ok()
.map(Duration::from_secs)
}
fn truncate(body: &str) -> String {
body.chars().take(MAX_ERROR_BODY).collect()
}
#[derive(Serialize)]
struct RequestCue<'a> {
number: usize,
text: &'a str,
}
#[derive(Deserialize)]
struct ReplyCue {
number: usize,
text: String,
}
#[derive(Serialize)]
struct ChatMessage {
role: &'static str,
content: String,
}
#[derive(Serialize)]
struct ChatRequest<'a> {
model: &'a str,
messages: Vec<ChatMessage>,
}
#[derive(Debug, Deserialize)]
struct ChatResponse {
choices: Vec<ChatChoice>,
}
#[derive(Debug, Deserialize)]
struct ChatChoice {
message: ChatReplyMessage,
finish_reason: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ChatReplyMessage {
content: Option<String>,
#[serde(default)]
refusal: Option<String>,
}
#[cfg(test)]
mod tests {
use arr_core::Language;
use super::OpenAi;
use crate::translate::Backend;
use crate::translate::{language_name, strip_code_fence, system_prompt};
fn backend() -> OpenAi {
OpenAi::new("gpt-4o-mini", super::OpenAiConfig { api_key: None }).expect("client builds")
}
#[test]
fn every_target_but_unverified_portuguese_is_supported() {
let backend = backend();
assert!(backend.supports(&Language::PortuguesePortugal));
assert!(backend.supports(&Language::PortugueseBrazil));
assert!(backend.supports(&Language::Other("en".to_owned())));
assert!(!backend.supports(&Language::PortugueseUnverified));
}
#[test]
fn the_prompt_tells_pt_pt_and_pt_br_apart() {
let source = Language::Other("en".to_owned());
let pt_pt = system_prompt(&source, &Language::PortuguesePortugal);
let pt_br = system_prompt(&source, &Language::PortugueseBrazil);
assert!(pt_pt.contains("European Portuguese"));
assert!(pt_pt.contains("comboio"));
assert!(pt_br.contains("Brazilian Portuguese"));
assert!(pt_br.contains("trem"));
assert_ne!(pt_pt, pt_br);
}
#[test]
fn the_prompt_asks_for_json_only() {
let prompt = system_prompt(
&Language::Other("en".to_owned()),
&Language::Other("fr".to_owned()),
);
assert!(prompt.contains("JSON"));
assert!(prompt.contains("never add, drop, merge or renumber"));
}
#[test]
fn language_names_read_like_prose_not_settings_rows() {
assert_eq!(
language_name(&Language::PortuguesePortugal),
"European Portuguese (pt-PT)"
);
assert_eq!(language_name(&Language::Other("en".to_owned())), "en");
}
#[test]
fn a_fenced_reply_is_unwrapped() {
assert_eq!(strip_code_fence("```json\n[1,2]\n```"), "[1,2]");
assert_eq!(strip_code_fence("```\n[1,2]\n```"), "[1,2]");
assert_eq!(strip_code_fence("[1,2]"), "[1,2]");
}
}