The base URL and model move into a cell the backend re-reads per request, so an operator can repoint it without a restart. Migration 0028 adds the two columns; DESIGN.md §15 calls both database rows. Construction never depends on the API key — llama.cpp serves without one.
509 lines
17 KiB
Rust
509 lines
17 KiB
Rust
//! The OpenAI-compatible translation backend (`DESIGN.md` §15, issue #191).
|
|
//!
|
|
//! One HTTP shape — the `/chat/completions` endpoint `OpenAI` defined — reaches
|
|
//! the largest number of options: `OpenAI` itself, `OpenRouter`, Groq, a
|
|
//! self-hosted gateway, or a local `llama.cpp` server. The model and base URL
|
|
//! live in an [`OpenAiEndpoint`] cell re-read per request rather than on
|
|
//! [`OpenAiConfig`]: both are database rows the operator edits from
|
|
//! `/settings`, and an edit reaches this running backend without a restart
|
|
//! (DESIGN.md §15, issues #198 and #220). Only the credential is bootstrap
|
|
//! config or environment, per §10 — and even that is optional, since a
|
|
//! self-hosted gateway or a local `llama.cpp` may not ask for one.
|
|
//!
|
|
//! The batch travels as a JSON array of `{number, text}` objects, and the
|
|
//! reply is asked for in the same shape: JSON round-trips a cue's embedded
|
|
//! newlines cleanly, where a line-oriented reply format could not tell a
|
|
//! multi-line cue from two cues. [`translate::translate`] chunks, validates
|
|
//! and reassembles; this module only builds one request and turns one reply
|
|
//! into [`TranslatedCue`]s. A refusal, a content-filtered reply or a reply cut
|
|
//! short by the token limit each fail with their own [`Error::Malformed`]
|
|
//! detail rather than reaching validation as a silently short batch.
|
|
|
|
use std::fmt;
|
|
use std::time::Duration;
|
|
|
|
use reqwest::{Client, StatusCode};
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use arr_core::Language;
|
|
|
|
use crate::openai_endpoint::OpenAiEndpoint;
|
|
use crate::translate::{
|
|
strip_code_fence, system_prompt, Backend, BackendId, Batch, Error, ProbeFuture, Result,
|
|
TranslateFuture, TranslatedCue,
|
|
};
|
|
|
|
const BACKEND_NAME: &str = "openai";
|
|
const USER_AGENT: &str = "arr v0.1.0";
|
|
const MAX_ERROR_BODY: usize = 512;
|
|
|
|
/// Credentials for an OpenAI-compatible endpoint (DESIGN.md §10).
|
|
///
|
|
/// Never persisted to the database — it reaches here from bootstrap config or
|
|
/// environment only. Optional because a self-hosted gateway (a local
|
|
/// `llama.cpp` server, say) may not require one.
|
|
#[derive(Clone, PartialEq, Eq)]
|
|
pub struct OpenAiConfig {
|
|
/// Sent as an `Authorization: Bearer` header when present.
|
|
pub api_key: Option<String>,
|
|
}
|
|
|
|
// Hand-written: a derived Debug would print the key.
|
|
impl fmt::Debug for OpenAiConfig {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
f.debug_struct("OpenAiConfig")
|
|
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
/// The OpenAI-compatible backend behind the [`Backend`] trait.
|
|
pub struct OpenAi {
|
|
http: Client,
|
|
endpoint: OpenAiEndpoint,
|
|
config: OpenAiConfig,
|
|
id: BackendId,
|
|
}
|
|
|
|
// Hand-written: the config must never reach a log line.
|
|
impl fmt::Debug for OpenAi {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
let endpoint = self.endpoint.read();
|
|
f.debug_struct("OpenAi")
|
|
.field("base_url", &endpoint.base_url.as_str())
|
|
.field("model", &endpoint.model)
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl OpenAi {
|
|
/// A client at the backend's own defaults, ready to be repointed through
|
|
/// [`Self::endpoint`] once the settings row has been read.
|
|
///
|
|
/// Construction never depends on the API key: `llama.cpp` serves without
|
|
/// authentication, so a base URL and no key is a valid configuration
|
|
/// (DESIGN.md §15).
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Fails if the HTTP client cannot be constructed.
|
|
pub fn new(config: OpenAiConfig) -> Result<Self> {
|
|
Self::with_endpoint(config, OpenAiEndpoint::new(None, None)?)
|
|
}
|
|
|
|
/// Point the client at another OpenAI-compatible endpoint — `OpenRouter`,
|
|
/// Groq, a self-hosted gateway, a local `llama.cpp` server, or `wiremock`
|
|
/// in tests — with a fixed model.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Same as [`Self::new`], plus a `base_url` that does not parse.
|
|
pub fn with_base_url(
|
|
model: impl Into<String>,
|
|
config: OpenAiConfig,
|
|
base_url: &str,
|
|
) -> Result<Self> {
|
|
let model = model.into();
|
|
Self::with_endpoint(
|
|
config,
|
|
OpenAiEndpoint::new(Some(base_url), Some(model.as_str()))?,
|
|
)
|
|
}
|
|
|
|
/// A client re-reading `endpoint` for every request, so the settings API
|
|
/// can repoint it without a restart (#220).
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Fails if the HTTP client cannot be constructed.
|
|
pub fn with_endpoint(config: OpenAiConfig, endpoint: OpenAiEndpoint) -> Result<Self> {
|
|
let id = BackendId::new(BACKEND_NAME);
|
|
let http = Client::builder().build().map_err(|err| Error::Transport {
|
|
backend: id.clone(),
|
|
source: Box::new(err),
|
|
})?;
|
|
Ok(Self {
|
|
http,
|
|
endpoint,
|
|
config,
|
|
id,
|
|
})
|
|
}
|
|
|
|
/// The cell this backend re-reads for every request.
|
|
///
|
|
/// Whoever constructs the backend — the daemon, at startup — hands a
|
|
/// clone of this to the settings API, which stores `openai_base_url` and
|
|
/// `openai_model` into it on every edit. Same shape as #219's
|
|
/// `Command::timeout_cell`.
|
|
#[must_use]
|
|
pub fn endpoint(&self) -> OpenAiEndpoint {
|
|
self.endpoint.clone()
|
|
}
|
|
|
|
async fn check_status(&self, response: reqwest::Response) -> Result<reqwest::Response> {
|
|
match response.status() {
|
|
StatusCode::OK => Ok(response),
|
|
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(Error::Unauthorized {
|
|
backend: self.id.clone(),
|
|
}),
|
|
StatusCode::TOO_MANY_REQUESTS => Err(Error::RateLimited {
|
|
backend: self.id.clone(),
|
|
retry_after: retry_after(&response),
|
|
}),
|
|
other => {
|
|
let body = response.text().await.unwrap_or_default();
|
|
Err(Error::Malformed {
|
|
backend: self.id.clone(),
|
|
detail: format!("status {}: {}", other.as_u16(), truncate(&body)),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn decode<T: for<'de> Deserialize<'de>>(&self, response: reqwest::Response) -> Result<T> {
|
|
let bytes = response.bytes().await.map_err(|err| Error::Transport {
|
|
backend: self.id.clone(),
|
|
source: Box::new(err),
|
|
})?;
|
|
serde_json::from_slice(&bytes).map_err(|err| Error::Malformed {
|
|
backend: self.id.clone(),
|
|
detail: err.to_string(),
|
|
})
|
|
}
|
|
|
|
async fn translate_inner(&self, batch: &Batch) -> Result<Vec<TranslatedCue>> {
|
|
let request_cues: Vec<RequestCue<'_>> = batch
|
|
.cues
|
|
.iter()
|
|
.map(|cue| RequestCue {
|
|
number: cue.number,
|
|
text: &cue.text,
|
|
})
|
|
.collect();
|
|
let user_content =
|
|
serde_json::to_string(&request_cues).map_err(|err| Error::Malformed {
|
|
backend: self.id.clone(),
|
|
detail: format!("could not encode batch: {err}"),
|
|
})?;
|
|
|
|
// Re-read per call: the settings API writes this cell when the
|
|
// operator edits `openai_base_url` or `openai_model` (#220).
|
|
let endpoint = self.endpoint.read();
|
|
let body = ChatRequest {
|
|
model: &endpoint.model,
|
|
messages: vec![
|
|
ChatMessage {
|
|
role: "system",
|
|
content: system_prompt(&batch.source, &batch.target),
|
|
},
|
|
ChatMessage {
|
|
role: "user",
|
|
content: user_content,
|
|
},
|
|
],
|
|
};
|
|
|
|
let url = endpoint
|
|
.base_url
|
|
.join("chat/completions")
|
|
.map_err(|err| Error::Malformed {
|
|
backend: self.id.clone(),
|
|
detail: format!("bad chat completions path: {err}"),
|
|
})?;
|
|
|
|
let mut request = self
|
|
.http
|
|
.post(url)
|
|
.header("User-Agent", USER_AGENT)
|
|
.json(&body);
|
|
if let Some(api_key) = self.config.api_key.as_deref() {
|
|
request = request.bearer_auth(api_key);
|
|
}
|
|
|
|
tracing::debug!(backend = %self.id, model = %endpoint.model, "OpenAI translate request");
|
|
let response = request.send().await.map_err(|err| Error::Transport {
|
|
backend: self.id.clone(),
|
|
source: Box::new(err),
|
|
})?;
|
|
let response = self.check_status(response).await?;
|
|
let reply: ChatResponse = self.decode(response).await?;
|
|
self.parse_reply(reply)
|
|
}
|
|
|
|
fn parse_reply(&self, reply: ChatResponse) -> Result<Vec<TranslatedCue>> {
|
|
let malformed = |detail: String| Error::Malformed {
|
|
backend: self.id.clone(),
|
|
detail,
|
|
};
|
|
|
|
let choice = reply
|
|
.choices
|
|
.into_iter()
|
|
.next()
|
|
.ok_or_else(|| malformed("no choices in reply".to_owned()))?;
|
|
|
|
if let Some(refusal) = choice.message.refusal.filter(|refusal| !refusal.is_empty()) {
|
|
return Err(malformed(format!("backend refused: {refusal}")));
|
|
}
|
|
match choice.finish_reason.as_deref() {
|
|
Some("content_filter") => {
|
|
return Err(malformed("reply blocked by content filter".to_owned()))
|
|
}
|
|
Some("length") => {
|
|
return Err(malformed(
|
|
"reply was truncated before it finished".to_owned(),
|
|
))
|
|
}
|
|
_ => {}
|
|
}
|
|
|
|
let content = choice
|
|
.message
|
|
.content
|
|
.filter(|content| !content.trim().is_empty())
|
|
.ok_or_else(|| malformed("empty reply".to_owned()))?;
|
|
|
|
let cues: Vec<ReplyCue> = serde_json::from_str(strip_code_fence(&content))
|
|
.map_err(|err| malformed(format!("reply was not the expected JSON: {err}")))?;
|
|
|
|
Ok(cues
|
|
.into_iter()
|
|
.map(|cue| TranslatedCue {
|
|
number: cue.number,
|
|
text: cue.text,
|
|
})
|
|
.collect())
|
|
}
|
|
}
|
|
|
|
impl Backend for OpenAi {
|
|
fn id(&self) -> BackendId {
|
|
self.id.clone()
|
|
}
|
|
|
|
/// True for every target except [`Language::PortugueseUnverified`]: an
|
|
/// unverified variant names no exact target to translate into, and an LLM
|
|
/// asked to hit one anyway would have to guess (DESIGN.md §15).
|
|
fn supports(&self, target: &Language) -> bool {
|
|
!matches!(target, Language::PortugueseUnverified)
|
|
}
|
|
|
|
fn translate<'a>(&'a self, batch: &'a Batch) -> TranslateFuture<'a> {
|
|
Box::pin(async move { self.translate_inner(batch).await })
|
|
}
|
|
|
|
/// `models` is a read-only listing: reachable and key accepted, nothing
|
|
/// spent (#200). Endpoints that need no key are probed the same way,
|
|
/// just without the header.
|
|
///
|
|
/// The base URL is read from the live cell, not from a start-up copy, so
|
|
/// the lamp judges whatever endpoint is configured right now (#220).
|
|
fn probe(&self) -> ProbeFuture<'_> {
|
|
Box::pin(async move {
|
|
let mut url = self.endpoint.read().base_url;
|
|
url.set_path(&format!("{}/models", url.path().trim_end_matches('/')));
|
|
let mut request = self.http.get(url);
|
|
if let Some(key) = &self.config.api_key {
|
|
request = request.bearer_auth(key);
|
|
}
|
|
match request.send().await {
|
|
Ok(response) => match response.status() {
|
|
StatusCode::OK => Ok(()),
|
|
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {
|
|
Err(Error::Unauthorized { backend: self.id() })
|
|
}
|
|
status => Err(Error::Malformed {
|
|
backend: self.id(),
|
|
detail: format!("status {}", status.as_u16()),
|
|
}),
|
|
},
|
|
Err(error) => Err(Error::Transport {
|
|
backend: self.id(),
|
|
source: Box::new(error),
|
|
}),
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
fn retry_after(response: &reqwest::Response) -> Option<Duration> {
|
|
response
|
|
.headers()
|
|
.get(reqwest::header::RETRY_AFTER)?
|
|
.to_str()
|
|
.ok()?
|
|
.trim()
|
|
.parse::<u64>()
|
|
.ok()
|
|
.map(Duration::from_secs)
|
|
}
|
|
|
|
fn truncate(body: &str) -> String {
|
|
body.chars().take(MAX_ERROR_BODY).collect()
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct RequestCue<'a> {
|
|
number: usize,
|
|
text: &'a str,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct ReplyCue {
|
|
number: usize,
|
|
text: String,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct ChatMessage {
|
|
role: &'static str,
|
|
content: String,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct ChatRequest<'a> {
|
|
model: &'a str,
|
|
messages: Vec<ChatMessage>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct ChatResponse {
|
|
choices: Vec<ChatChoice>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct ChatChoice {
|
|
message: ChatReplyMessage,
|
|
finish_reason: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct ChatReplyMessage {
|
|
content: Option<String>,
|
|
#[serde(default)]
|
|
refusal: Option<String>,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use arr_core::Language;
|
|
|
|
use super::OpenAi;
|
|
use crate::openai_endpoint::DEFAULT_MODEL;
|
|
use crate::translate::Backend;
|
|
use crate::translate::{language_name, strip_code_fence, system_prompt};
|
|
|
|
fn backend() -> OpenAi {
|
|
OpenAi::new(super::OpenAiConfig { api_key: None }).expect("client builds")
|
|
}
|
|
|
|
/// #220: the backend holds the cell, not a copy — a repoint between two
|
|
/// calls is what the settings handler relies on.
|
|
#[test]
|
|
fn the_backend_follows_its_endpoint_cell() {
|
|
let backend = backend();
|
|
let cell = backend.endpoint();
|
|
assert_eq!(cell.model(), DEFAULT_MODEL);
|
|
cell.set(Some("http://box:8080/v1/"), Some("llama"))
|
|
.expect("repoint");
|
|
assert_eq!(backend.endpoint.model(), "llama");
|
|
assert_eq!(backend.endpoint.base_url(), "http://box:8080/v1/");
|
|
}
|
|
|
|
#[test]
|
|
fn every_target_but_unverified_portuguese_is_supported() {
|
|
let backend = backend();
|
|
assert!(backend.supports(&Language::PortuguesePortugal));
|
|
assert!(backend.supports(&Language::PortugueseBrazil));
|
|
assert!(backend.supports(&Language::Other("en".to_owned())));
|
|
assert!(!backend.supports(&Language::PortugueseUnverified));
|
|
}
|
|
|
|
#[test]
|
|
fn the_prompt_tells_pt_pt_and_pt_br_apart() {
|
|
let source = Language::Other("en".to_owned());
|
|
|
|
let pt_pt = system_prompt(&source, &Language::PortuguesePortugal);
|
|
let pt_br = system_prompt(&source, &Language::PortugueseBrazil);
|
|
|
|
assert!(pt_pt.contains("European Portuguese"));
|
|
assert!(pt_pt.contains("comboio"));
|
|
assert!(pt_br.contains("Brazilian Portuguese"));
|
|
assert!(pt_br.contains("trem"));
|
|
assert_ne!(pt_pt, pt_br);
|
|
}
|
|
|
|
#[test]
|
|
fn the_prompt_asks_for_json_only() {
|
|
let prompt = system_prompt(
|
|
&Language::Other("en".to_owned()),
|
|
&Language::Other("fr".to_owned()),
|
|
);
|
|
assert!(prompt.contains("JSON"));
|
|
assert!(prompt.contains("never add, drop, merge or renumber"));
|
|
}
|
|
|
|
#[test]
|
|
fn language_names_read_like_prose_not_settings_rows() {
|
|
assert_eq!(
|
|
language_name(&Language::PortuguesePortugal),
|
|
"European Portuguese (pt-PT)"
|
|
);
|
|
assert_eq!(language_name(&Language::Other("en".to_owned())), "en");
|
|
}
|
|
|
|
#[test]
|
|
fn a_fenced_reply_is_unwrapped() {
|
|
assert_eq!(strip_code_fence("```json\n[1,2]\n```"), "[1,2]");
|
|
assert_eq!(strip_code_fence("```\n[1,2]\n```"), "[1,2]");
|
|
assert_eq!(strip_code_fence("[1,2]"), "[1,2]");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_probe_lists_models_and_judges_the_key() {
|
|
use wiremock::matchers::{header, method, path};
|
|
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
|
|
let server = MockServer::start().await;
|
|
Mock::given(method("GET"))
|
|
.and(path("/v1/models"))
|
|
.and(header("Authorization", "Bearer test-key"))
|
|
.respond_with(ResponseTemplate::new(200).set_body_string("{}"))
|
|
.mount(&server)
|
|
.await;
|
|
OpenAi::with_base_url(
|
|
"gpt-4o-mini",
|
|
super::OpenAiConfig {
|
|
api_key: Some("test-key".to_owned()),
|
|
},
|
|
&format!("{}/v1", server.uri()),
|
|
)
|
|
.expect("client builds")
|
|
.probe()
|
|
.await
|
|
.expect("a models answer is a lit lamp");
|
|
|
|
server.reset().await;
|
|
Mock::given(method("GET"))
|
|
.respond_with(ResponseTemplate::new(401))
|
|
.mount(&server)
|
|
.await;
|
|
let error = OpenAi::with_base_url(
|
|
"gpt-4o-mini",
|
|
super::OpenAiConfig {
|
|
api_key: Some("test-key".to_owned()),
|
|
},
|
|
&format!("{}/v1", server.uri()),
|
|
)
|
|
.expect("client builds")
|
|
.probe()
|
|
.await
|
|
.expect_err("bad key");
|
|
assert!(
|
|
matches!(error, crate::translate::Error::Unauthorized { .. }),
|
|
"got {error:?}"
|
|
);
|
|
}
|
|
}
|