First Version
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "llm-client"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
core-api = { path = "../core-api" }
|
||||
reqwest = { version = "0.13", default-features = false, features = ["rustls-no-provider", "charset", "http2", "system-proxy", "json"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
async-trait = "0.1"
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
@@ -0,0 +1,366 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
use crate::{ChatOptions, ChatResponse, ChatbotClient, LlmRawMeta, LlmTurn, Message, Role, ToolCall, headers_to_json, redact_key};
|
||||
|
||||
const DEFAULT_BASE_URL: &str = "https://api.anthropic.com";
|
||||
const ANTHROPIC_VERSION: &str = "2023-06-01";
|
||||
|
||||
pub struct AnthropicClient {
|
||||
base_url: String,
|
||||
api_key: String,
|
||||
/// Extra top-level request-body keys merged into every request (e.g. the
|
||||
/// `thinking` config for extended reasoning). See `apply_extra`.
|
||||
extra_body: Option<Value>,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl AnthropicClient {
|
||||
pub fn new(api_key: impl Into<String>) -> Self {
|
||||
Self::with_base_url(DEFAULT_BASE_URL, api_key)
|
||||
}
|
||||
|
||||
pub fn with_base_url(base_url: impl Into<String>, api_key: impl Into<String>) -> Self {
|
||||
Self {
|
||||
base_url: base_url.into(),
|
||||
api_key: api_key.into(),
|
||||
extra_body: None,
|
||||
http: reqwest::Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Like `new` but with extra request-body keys (e.g. `{"thinking": {...}}`).
|
||||
pub fn with_extra_body(api_key: impl Into<String>, extra_body: Option<Value>) -> Self {
|
||||
Self {
|
||||
base_url: DEFAULT_BASE_URL.to_string(),
|
||||
api_key: api_key.into(),
|
||||
extra_body,
|
||||
http: reqwest::Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges `extra_body` into `body` and enforces Anthropic's extended-thinking
|
||||
/// constraints: when `thinking` is enabled, `temperature` is not allowed and
|
||||
/// `max_tokens` must be strictly greater than `budget_tokens`.
|
||||
fn apply_extra(&self, body: &mut Value) {
|
||||
let Some(extra) = self.extra_body.as_ref().and_then(|v| v.as_object()) else { return };
|
||||
let Some(obj) = body.as_object_mut() else { return };
|
||||
for (k, v) in extra {
|
||||
obj.insert(k.clone(), v.clone());
|
||||
}
|
||||
if obj.get("thinking").map(|t| t["type"] == json!("enabled")).unwrap_or(false) {
|
||||
obj.remove("temperature");
|
||||
let budget = obj["thinking"]["budget_tokens"].as_i64().unwrap_or(0);
|
||||
let cur_max = obj.get("max_tokens").and_then(|v| v.as_i64()).unwrap_or(4096);
|
||||
if budget > 0 && cur_max <= budget {
|
||||
obj.insert("max_tokens".to_string(), json!(budget + 4096));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts OpenAI-format tool definitions to Anthropic format.
|
||||
/// OpenAI: { "type": "function", "function": { "name", "description", "parameters" } }
|
||||
/// Anthropic: { "name", "description", "input_schema" }
|
||||
fn convert_tools(tools: &[Value]) -> Vec<Value> {
|
||||
tools
|
||||
.iter()
|
||||
.filter_map(|t| {
|
||||
let func = &t["function"];
|
||||
let name = func["name"].as_str()?;
|
||||
Some(json!({
|
||||
"name": name,
|
||||
"description": func["description"].as_str().unwrap_or(""),
|
||||
"input_schema": func["parameters"],
|
||||
}))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Converts OpenAI-format message array to Anthropic format.
|
||||
///
|
||||
/// Key differences:
|
||||
/// - System messages are skipped (extracted separately).
|
||||
/// - Assistant messages with `tool_calls` become content arrays with `tool_use` blocks.
|
||||
/// - `tool` role messages are grouped into `user` messages with `tool_result` blocks.
|
||||
fn convert_messages(messages: &[Value]) -> Vec<Value> {
|
||||
let mut out: Vec<Value> = Vec::new();
|
||||
let mut i = 0;
|
||||
|
||||
while i < messages.len() {
|
||||
let msg = &messages[i];
|
||||
let role = msg["role"].as_str().unwrap_or("");
|
||||
|
||||
match role {
|
||||
"system" => { i += 1; }
|
||||
|
||||
"user" => {
|
||||
out.push(json!({
|
||||
"role": "user",
|
||||
"content": msg["content"].as_str().unwrap_or(""),
|
||||
}));
|
||||
i += 1;
|
||||
}
|
||||
|
||||
"assistant" => {
|
||||
if let Some(tool_calls) = msg["tool_calls"].as_array() {
|
||||
let mut content: Vec<Value> = Vec::new();
|
||||
|
||||
let text = msg["content"].as_str().unwrap_or("");
|
||||
if !text.is_empty() {
|
||||
content.push(json!({ "type": "text", "text": text }));
|
||||
}
|
||||
|
||||
for tc in tool_calls {
|
||||
let id = tc["id"].as_str().unwrap_or("");
|
||||
let name = tc["function"]["name"].as_str().unwrap_or("");
|
||||
let args_str = tc["function"]["arguments"].as_str().unwrap_or("{}");
|
||||
let input: Value = serde_json::from_str(args_str)
|
||||
.unwrap_or(Value::Object(Default::default()));
|
||||
|
||||
content.push(json!({
|
||||
"type": "tool_use",
|
||||
"id": id,
|
||||
"name": name,
|
||||
"input": input,
|
||||
}));
|
||||
}
|
||||
|
||||
out.push(json!({ "role": "assistant", "content": content }));
|
||||
} else {
|
||||
out.push(json!({
|
||||
"role": "assistant",
|
||||
"content": msg["content"].as_str().unwrap_or(""),
|
||||
}));
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
"tool" => {
|
||||
// Group all consecutive tool-result messages into a single user message.
|
||||
let mut results: Vec<Value> = Vec::new();
|
||||
while i < messages.len() && messages[i]["role"].as_str() == Some("tool") {
|
||||
let tm = &messages[i];
|
||||
results.push(json!({
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tm["tool_call_id"].as_str().unwrap_or(""),
|
||||
"content": tm["content"].as_str().unwrap_or(""),
|
||||
}));
|
||||
i += 1;
|
||||
}
|
||||
out.push(json!({ "role": "user", "content": results }));
|
||||
}
|
||||
|
||||
_ => { i += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChatbotClient for AnthropicClient {
|
||||
async fn chat(
|
||||
&self,
|
||||
messages: &[Message],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<ChatResponse> {
|
||||
// Merge all system-role messages into a single `system:` parameter.
|
||||
let system: Option<String> = {
|
||||
let parts: Vec<&str> = messages
|
||||
.iter()
|
||||
.filter(|m| m.role == Role::System)
|
||||
.map(|m| m.content.as_str())
|
||||
.collect();
|
||||
if parts.is_empty() { None } else { Some(parts.join("\n\n---\n\n")) }
|
||||
};
|
||||
|
||||
let msgs: Vec<Value> = messages
|
||||
.iter()
|
||||
.filter(|m| m.role != Role::System)
|
||||
.map(|m| {
|
||||
let role = match m.role {
|
||||
Role::User => "user",
|
||||
Role::Assistant => "assistant",
|
||||
Role::System => unreachable!(),
|
||||
};
|
||||
json!({ "role": role, "content": m.content })
|
||||
})
|
||||
.collect();
|
||||
|
||||
let max_tokens = options.max_tokens.unwrap_or(4096);
|
||||
let mut body = json!({
|
||||
"model": options.model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": msgs,
|
||||
});
|
||||
|
||||
if let Some(sys) = system { body["system"] = sys.into(); }
|
||||
if let Some(t) = options.temperature { body["temperature"] = t.into(); }
|
||||
self.apply_extra(&mut body);
|
||||
|
||||
let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
|
||||
debug!(model = %options.model, "anthropic: sending chat request");
|
||||
trace!(body = %body, "anthropic: chat request body");
|
||||
|
||||
let resp: Value = self
|
||||
.http
|
||||
.post(&url)
|
||||
.header("x-api-key", &self.api_key)
|
||||
.header("anthropic-version", ANTHROPIC_VERSION)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
let content = resp["content"]
|
||||
.as_array()
|
||||
.and_then(|arr| arr.first())
|
||||
.and_then(|block| block["text"].as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing content in Anthropic response"))?
|
||||
.to_string();
|
||||
|
||||
let input_tokens = resp["usage"]["input_tokens"].as_u64().map(|n| n as u32);
|
||||
let output_tokens = resp["usage"]["output_tokens"].as_u64().map(|n| n as u32);
|
||||
let cache_read_tokens = resp["usage"]["cache_read_input_tokens"].as_u64().map(|n| n as u32);
|
||||
let cache_creation_tokens = resp["usage"]["cache_creation_input_tokens"].as_u64().map(|n| n as u32);
|
||||
info!(model = %options.model, ?input_tokens, ?output_tokens, "anthropic: chat response received");
|
||||
|
||||
let cost = self.extract_cost(&resp);
|
||||
Ok(ChatResponse { content, input_tokens, output_tokens, truncated: false, reasoning_content: None, cache_read_tokens, cache_creation_tokens, cost })
|
||||
}
|
||||
|
||||
async fn chat_with_tools(
|
||||
&self,
|
||||
messages: &[Value],
|
||||
tools: &[Value],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<LlmTurn> {
|
||||
self.chat_with_tools_raw(messages, tools, options).await.map(|(t, _)| t)
|
||||
}
|
||||
|
||||
async fn chat_with_tools_raw(
|
||||
&self,
|
||||
messages: &[Value],
|
||||
tools: &[Value],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<(LlmTurn, Option<LlmRawMeta>)> {
|
||||
// Collect ALL system-role messages (main prompt, mid-conversation
|
||||
// summary, tail_reminder) and merge them into a single `system:`
|
||||
// string. The Anthropic API only accepts a single system parameter;
|
||||
// mid-conversation system messages generated by build_openai_messages
|
||||
// are intentionally used for injecting compaction summaries and tail
|
||||
// reminders — they must not be silently dropped.
|
||||
let system: Option<String> = {
|
||||
let parts: Vec<&str> = messages
|
||||
.iter()
|
||||
.filter(|m| m["role"].as_str() == Some("system"))
|
||||
.filter_map(|m| m["content"].as_str())
|
||||
.collect();
|
||||
if parts.is_empty() { None } else { Some(parts.join("\n\n---\n\n")) }
|
||||
};
|
||||
|
||||
let anthropic_messages = Self::convert_messages(messages);
|
||||
let anthropic_tools = Self::convert_tools(tools);
|
||||
|
||||
let max_tokens = options.max_tokens.unwrap_or(4096);
|
||||
let mut body = json!({
|
||||
"model": options.model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": anthropic_messages,
|
||||
"tools": anthropic_tools,
|
||||
});
|
||||
|
||||
if let Some(sys) = system { body["system"] = sys.into(); }
|
||||
if let Some(t) = options.temperature { body["temperature"] = t.into(); }
|
||||
self.apply_extra(&mut body);
|
||||
|
||||
let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
|
||||
debug!(model = %options.model, tools = tools.len(), "anthropic: sending chat_with_tools request");
|
||||
trace!(body = %body, "anthropic: chat_with_tools request body");
|
||||
|
||||
// Capture request metadata for logging.
|
||||
let request_body = body.clone();
|
||||
let request_headers = json!({
|
||||
"x-api-key": redact_key(&self.api_key),
|
||||
"anthropic-version": ANTHROPIC_VERSION,
|
||||
"content-type": "application/json",
|
||||
});
|
||||
|
||||
let http_resp = self
|
||||
.http
|
||||
.post(&url)
|
||||
.header("x-api-key", &self.api_key)
|
||||
.header("anthropic-version", ANTHROPIC_VERSION)
|
||||
.header("X-Title", core_api::APP_NAME)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
let response_headers = headers_to_json(http_resp.headers());
|
||||
let resp_text = http_resp.text().await?;
|
||||
let resp: Value = serde_json::from_str(&resp_text)
|
||||
.map_err(|e| anyhow::anyhow!("anthropic: failed to parse response JSON: {e}\nbody: {resp_text}"))?;
|
||||
let response_body: Value = serde_json::from_str(&resp_text).unwrap_or(Value::Null);
|
||||
|
||||
let raw_meta = LlmRawMeta {
|
||||
request_headers: Some(request_headers),
|
||||
request_body: Some(request_body),
|
||||
response_headers: Some(response_headers),
|
||||
response_body: Some(response_body),
|
||||
};
|
||||
|
||||
let stop_reason = resp["stop_reason"].as_str().unwrap_or("");
|
||||
let input_tokens = resp["usage"]["input_tokens"].as_u64().map(|n| n as u32);
|
||||
let output_tokens = resp["usage"]["output_tokens"].as_u64().map(|n| n as u32);
|
||||
let cache_read_tokens = resp["usage"]["cache_read_input_tokens"].as_u64().map(|n| n as u32);
|
||||
let cache_creation_tokens = resp["usage"]["cache_creation_input_tokens"].as_u64().map(|n| n as u32);
|
||||
let content_blocks = resp["content"].as_array().cloned().unwrap_or_default();
|
||||
let cost = self.extract_cost(&resp);
|
||||
info!(model = %options.model, ?input_tokens, ?output_tokens, stop_reason, "anthropic: chat_with_tools response received");
|
||||
if stop_reason == "max_tokens" {
|
||||
warn!(model = %options.model, ?output_tokens, "anthropic: response truncated (max_tokens reached)");
|
||||
}
|
||||
|
||||
let has_tool_use = content_blocks.iter().any(|b| b["type"].as_str() == Some("tool_use"));
|
||||
|
||||
// Check content blocks directly: Anthropic sometimes returns stop_reason "end_turn"
|
||||
// even when tool_use blocks are present, so stop_reason alone is not reliable.
|
||||
let turn = if stop_reason == "tool_use" || has_tool_use {
|
||||
let text: String = content_blocks
|
||||
.iter()
|
||||
.filter(|b| b["type"].as_str() == Some("text"))
|
||||
.filter_map(|b| b["text"].as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let calls: Vec<ToolCall> = content_blocks
|
||||
.iter()
|
||||
.filter(|b| b["type"].as_str() == Some("tool_use"))
|
||||
.map(|b| ToolCall {
|
||||
id: b["id"].as_str().unwrap_or("").to_string(),
|
||||
name: b["name"].as_str().unwrap_or("").to_string(),
|
||||
arguments: b["input"].clone(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
LlmTurn::ToolCalls { content: text, calls, input_tokens, output_tokens, reasoning_content: None, cache_read_tokens, cache_creation_tokens, cost }
|
||||
} else {
|
||||
let content = content_blocks
|
||||
.iter()
|
||||
.find(|b| b["type"].as_str() == Some("text"))
|
||||
.and_then(|b| b["text"].as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let truncated = stop_reason == "max_tokens";
|
||||
LlmTurn::Message(ChatResponse { content, input_tokens, output_tokens, truncated, reasoning_content: None, cache_read_tokens, cache_creation_tokens, cost })
|
||||
};
|
||||
|
||||
Ok((turn, Some(raw_meta)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
pub mod anthropic;
|
||||
pub mod lm_studio;
|
||||
pub mod ollama;
|
||||
pub mod openai;
|
||||
|
||||
// Re-export the trait and all associated types from core-api so existing
|
||||
// callers that import from `llm_client` continue to work unchanged.
|
||||
pub use core_api::chatbot::{
|
||||
ChatOptions, ChatResponse, ChatbotClient, LlmRawMeta, LlmTurn, Message, Role, ToolCall,
|
||||
};
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// Converts a reqwest `HeaderMap` into a `serde_json::Value` object.
|
||||
pub fn headers_to_json(headers: &reqwest::header::HeaderMap) -> Value {
|
||||
let map: serde_json::Map<String, Value> = headers
|
||||
.iter()
|
||||
.map(|(k, v)| (
|
||||
k.as_str().to_string(),
|
||||
v.to_str().unwrap_or("<binary>").into(),
|
||||
))
|
||||
.collect();
|
||||
Value::Object(map)
|
||||
}
|
||||
|
||||
/// Returns a redacted preview of an API key: first 7 chars + "***".
|
||||
pub fn redact_key(key: &str) -> String {
|
||||
if key.len() > 7 {
|
||||
format!("{}***", &key[..7])
|
||||
} else {
|
||||
"***".to_string()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{ChatOptions, ChatResponse, ChatbotClient, LlmRawMeta, LlmTurn, Message, openai::OpenAiClient};
|
||||
|
||||
/// LM Studio client.
|
||||
///
|
||||
/// LM Studio exposes an OpenAI-compatible `/v1` endpoint, so this is a thin
|
||||
/// wrapper that defaults to `http://localhost:1234/v1` and requires no API key.
|
||||
pub struct LmStudioClient {
|
||||
inner: OpenAiClient,
|
||||
}
|
||||
|
||||
impl LmStudioClient {
|
||||
/// `base_url` defaults to `http://localhost:1234/v1` if `None`.
|
||||
pub fn new(base_url: Option<impl Into<String>>) -> Self {
|
||||
let url = base_url
|
||||
.map(|u| u.into())
|
||||
.unwrap_or_else(|| "http://localhost:1234/v1".to_string());
|
||||
Self { inner: OpenAiClient::new(url, "", None, false) }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChatbotClient for LmStudioClient {
|
||||
async fn chat(
|
||||
&self,
|
||||
messages: &[Message],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<ChatResponse> {
|
||||
self.inner.chat(messages, options).await
|
||||
}
|
||||
|
||||
async fn chat_with_tools(
|
||||
&self,
|
||||
messages: &[Value],
|
||||
tools: &[Value],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<LlmTurn> {
|
||||
self.inner.chat_with_tools(messages, tools, options).await
|
||||
}
|
||||
|
||||
async fn chat_with_tools_raw(
|
||||
&self,
|
||||
messages: &[Value],
|
||||
tools: &[Value],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<(LlmTurn, Option<LlmRawMeta>)> {
|
||||
self.inner.chat_with_tools_raw(messages, tools, options).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{ChatOptions, ChatResponse, ChatbotClient, Message, Role};
|
||||
|
||||
/// Ollama client using the native `/api/chat` endpoint.
|
||||
///
|
||||
/// Defaults to `http://localhost:11434`. No API key required.
|
||||
pub struct OllamaClient {
|
||||
base_url: String,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl OllamaClient {
|
||||
/// `base_url` defaults to `http://localhost:11434` if `None`.
|
||||
pub fn new(base_url: Option<impl Into<String>>) -> Self {
|
||||
let url = base_url
|
||||
.map(|u| u.into())
|
||||
.unwrap_or_else(|| "http://localhost:11434".to_string());
|
||||
Self { base_url: url, http: reqwest::Client::new() }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChatbotClient for OllamaClient {
|
||||
async fn chat(
|
||||
&self,
|
||||
messages: &[Message],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<ChatResponse> {
|
||||
let msgs: Vec<Value> = messages
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let role = match m.role {
|
||||
Role::System => "system",
|
||||
Role::User => "user",
|
||||
Role::Assistant => "assistant",
|
||||
};
|
||||
json!({ "role": role, "content": m.content })
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut options_obj = json!({});
|
||||
if let Some(t) = options.temperature { options_obj["temperature"] = t.into(); }
|
||||
if let Some(n) = options.max_tokens { options_obj["num_predict"] = n.into(); }
|
||||
|
||||
let body = json!({
|
||||
"model": options.model,
|
||||
"messages": msgs,
|
||||
"stream": false,
|
||||
"options": options_obj,
|
||||
});
|
||||
|
||||
let url = format!("{}/api/chat", self.base_url.trim_end_matches('/'));
|
||||
|
||||
let resp: Value = self
|
||||
.http
|
||||
.post(&url)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
let content = resp["message"]["content"]
|
||||
.as_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing content in Ollama response"))?
|
||||
.to_string();
|
||||
|
||||
let input_tokens = resp["prompt_eval_count"].as_u64().map(|n| n as u32);
|
||||
let output_tokens = resp["eval_count"].as_u64().map(|n| n as u32);
|
||||
|
||||
Ok(ChatResponse { content, input_tokens, output_tokens, truncated: false, reasoning_content: None, cache_read_tokens: None, cache_creation_tokens: None, cost: None })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
use crate::{ChatOptions, ChatResponse, ChatbotClient, LlmRawMeta, LlmTurn, Message, Role, ToolCall, headers_to_json, redact_key};
|
||||
use core_api::APP_NAME;
|
||||
|
||||
/// OpenAI ChatGPT client (also compatible with any OpenAI-spec endpoint).
|
||||
pub struct OpenAiClient {
|
||||
base_url: String,
|
||||
api_key: String,
|
||||
extra_params: Option<serde_json::Value>,
|
||||
/// When true, Anthropic-compatible prompt-caching hints are injected:
|
||||
/// - `anthropic-beta: prompt-caching-2024-07-31` header is sent.
|
||||
/// - The last tool definition is tagged with `cache_control: {"type":"ephemeral"}`.
|
||||
/// - System message content is expected to already be a content array with
|
||||
/// `cache_control` on the static block (set by `build_openai_messages`).
|
||||
/// Used for OpenRouter when routing to Anthropic models.
|
||||
enable_prompt_cache: bool,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl OpenAiClient {
|
||||
pub fn new(base_url: impl Into<String>, api_key: impl Into<String>, extra_params: Option<serde_json::Value>, enable_prompt_cache: bool) -> Self {
|
||||
Self {
|
||||
base_url: base_url.into(),
|
||||
api_key: api_key.into(),
|
||||
extra_params,
|
||||
enable_prompt_cache,
|
||||
http: reqwest::Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges `extra_params` (if any) into `body`. Only top-level object keys are merged.
|
||||
fn apply_extra(&self, body: &mut serde_json::Value) {
|
||||
if let Some(serde_json::Value::Object(extra)) = &self.extra_params {
|
||||
if let Some(b) = body.as_object_mut() {
|
||||
for (k, v) in extra {
|
||||
b.insert(k.clone(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn url(&self) -> String {
|
||||
format!("{}/chat/completions", self.base_url.trim_end_matches('/'))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChatbotClient for OpenAiClient {
|
||||
async fn chat(
|
||||
&self,
|
||||
messages: &[Message],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<ChatResponse> {
|
||||
let msgs: Vec<Value> = messages
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let role = match m.role {
|
||||
Role::System => "system",
|
||||
Role::User => "user",
|
||||
Role::Assistant => "assistant",
|
||||
};
|
||||
json!({ "role": role, "content": m.content })
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut body = json!({
|
||||
"model": options.model,
|
||||
"messages": msgs,
|
||||
});
|
||||
|
||||
if let Some(t) = options.max_tokens { body["max_tokens"] = t.into(); }
|
||||
if let Some(t) = options.temperature { body["temperature"] = t.into(); }
|
||||
self.apply_extra(&mut body);
|
||||
|
||||
debug!(model = %options.model, "openai: sending chat request");
|
||||
trace!(body = %body, "openai: chat request body");
|
||||
|
||||
let resp: Value = self
|
||||
.http
|
||||
.post(self.url())
|
||||
.bearer_auth(&self.api_key)
|
||||
.header("X-Title", APP_NAME)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
let content = match resp["choices"][0]["message"]["content"].as_str() {
|
||||
Some(s) => s.to_string(),
|
||||
None => {
|
||||
warn!(raw_response = %resp, "openai: chat() response has null content");
|
||||
String::new()
|
||||
}
|
||||
};
|
||||
|
||||
let input_tokens = resp["usage"]["prompt_tokens"].as_u64().map(|n| n as u32);
|
||||
let output_tokens = resp["usage"]["completion_tokens"].as_u64().map(|n| n as u32);
|
||||
let cache_read_tokens = resp["usage"]["prompt_tokens_details"]["cached_tokens"].as_u64().map(|n| n as u32);
|
||||
let truncated = resp["choices"][0]["finish_reason"].as_str() == Some("length");
|
||||
let cost = self.extract_cost(&resp);
|
||||
info!(model = %options.model, ?input_tokens, ?output_tokens, ?cost, truncated, "openai: chat response received");
|
||||
|
||||
Ok(ChatResponse { content, input_tokens, output_tokens, truncated, reasoning_content: None, cache_read_tokens, cache_creation_tokens: None, cost })
|
||||
}
|
||||
|
||||
async fn chat_with_tools(
|
||||
&self,
|
||||
messages: &[Value],
|
||||
tools: &[Value],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<LlmTurn> {
|
||||
self.chat_with_tools_raw(messages, tools, options).await.map(|(t, _)| t)
|
||||
}
|
||||
|
||||
async fn chat_with_tools_raw(
|
||||
&self,
|
||||
messages: &[Value],
|
||||
tools: &[Value],
|
||||
options: &ChatOptions,
|
||||
) -> anyhow::Result<(LlmTurn, Option<LlmRawMeta>)> {
|
||||
let mut body = json!({
|
||||
"model": options.model,
|
||||
"messages": messages,
|
||||
});
|
||||
|
||||
if !tools.is_empty() {
|
||||
// When prompt caching is enabled, tag the last tool with cache_control
|
||||
// so the entire tools array is included in the Anthropic KV cache prefix.
|
||||
let tools_value: Value = if self.enable_prompt_cache {
|
||||
let mut tagged = tools.to_vec();
|
||||
if let Some(last) = tagged.last_mut() {
|
||||
last["cache_control"] = json!({"type": "ephemeral"});
|
||||
}
|
||||
tagged.into()
|
||||
} else {
|
||||
tools.into()
|
||||
};
|
||||
body["tools"] = tools_value;
|
||||
body["tool_choice"] = "auto".into();
|
||||
}
|
||||
|
||||
if let Some(t) = options.max_tokens { body["max_tokens"] = t.into(); }
|
||||
if let Some(t) = options.temperature { body["temperature"] = t.into(); }
|
||||
self.apply_extra(&mut body);
|
||||
|
||||
debug!(model = %options.model, tools = tools.len(), prompt_cache = self.enable_prompt_cache, "openai: sending chat_with_tools request");
|
||||
trace!(body = %body, "openai: chat_with_tools request body");
|
||||
|
||||
// Capture request metadata for logging.
|
||||
let mut logged_headers = json!({
|
||||
"authorization": format!("Bearer {}", redact_key(&self.api_key)),
|
||||
"content-type": "application/json",
|
||||
});
|
||||
if self.enable_prompt_cache {
|
||||
logged_headers["anthropic-beta"] = "prompt-caching-2024-07-31".into();
|
||||
}
|
||||
let request_body = body.clone();
|
||||
let request_headers = logged_headers;
|
||||
|
||||
let mut req = self.http.post(self.url()).bearer_auth(&self.api_key).header("X-Title", APP_NAME);
|
||||
if self.enable_prompt_cache {
|
||||
req = req.header("anthropic-beta", "prompt-caching-2024-07-31");
|
||||
}
|
||||
let http_resp = req
|
||||
.json(&body)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response_headers = headers_to_json(http_resp.headers());
|
||||
let status = http_resp.status();
|
||||
let resp_text = http_resp.text().await?;
|
||||
|
||||
if !status.is_success() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"openai: HTTP {status} from {url}\nbody: {resp_text}",
|
||||
url = self.url(),
|
||||
));
|
||||
}
|
||||
|
||||
let resp: Value = serde_json::from_str(&resp_text)
|
||||
.map_err(|e| anyhow::anyhow!("openai: failed to parse response JSON: {e}\nbody: {resp_text}"))?;
|
||||
let response_body: Value = serde_json::from_str(&resp_text).unwrap_or(Value::Null);
|
||||
|
||||
let raw_meta = LlmRawMeta {
|
||||
request_headers: Some(request_headers),
|
||||
request_body: Some(request_body),
|
||||
response_headers: Some(response_headers),
|
||||
response_body: Some(response_body),
|
||||
};
|
||||
|
||||
let input_tokens = resp["usage"]["prompt_tokens"].as_u64().map(|n| n as u32);
|
||||
let output_tokens = resp["usage"]["completion_tokens"].as_u64().map(|n| n as u32);
|
||||
let cache_read_tokens = resp["usage"]["prompt_tokens_details"]["cached_tokens"].as_u64().map(|n| n as u32);
|
||||
let cost = self.extract_cost(&resp);
|
||||
|
||||
let choice = &resp["choices"][0];
|
||||
let message = &choice["message"];
|
||||
let finish = choice["finish_reason"].as_str().unwrap_or("stop");
|
||||
info!(model = %options.model, ?input_tokens, ?output_tokens, finish_reason = finish, "openai: chat_with_tools response received");
|
||||
if finish == "length" {
|
||||
warn!(model = %options.model, ?output_tokens, "openai: response truncated (max_tokens reached)");
|
||||
}
|
||||
|
||||
// Thinking/reasoning content varies by provider:
|
||||
// - DeepSeek: "reasoning_content" (must be echoed back on subsequent turns, even as "")
|
||||
// - MiniMax M3 and others: "reasoning"
|
||||
// We normalize to a single field and echo under both names in message_builder.
|
||||
let reasoning_content = message["reasoning_content"].as_str()
|
||||
.or_else(|| message["reasoning"].as_str())
|
||||
.map(str::to_string);
|
||||
|
||||
let tool_calls_array = message["tool_calls"].as_array().filter(|a| !a.is_empty());
|
||||
|
||||
// Some models (e.g. Qwen via OpenRouter) return finish_reason "stop" even when
|
||||
// tool_calls are present, so check the array directly rather than relying on finish_reason.
|
||||
let turn = if finish == "tool_calls" || tool_calls_array.is_some() {
|
||||
let content = message["content"].as_str().unwrap_or("").to_string();
|
||||
|
||||
let calls = tool_calls_array
|
||||
.ok_or_else(|| anyhow::anyhow!("finish_reason=tool_calls but tool_calls array missing or empty"))?
|
||||
.iter()
|
||||
.map(|tc| {
|
||||
let id = tc["id"].as_str().unwrap_or("").to_string();
|
||||
let name = tc["function"]["name"].as_str().unwrap_or("").to_string();
|
||||
let args: Value = tc["function"]["arguments"]
|
||||
.as_str()
|
||||
.and_then(|s| serde_json::from_str(s).ok())
|
||||
.unwrap_or(Value::Object(Default::default()));
|
||||
ToolCall { id, name, arguments: args }
|
||||
})
|
||||
.collect();
|
||||
|
||||
LlmTurn::ToolCalls { content, calls, input_tokens, output_tokens, reasoning_content, cache_read_tokens, cache_creation_tokens: None, cost }
|
||||
} else {
|
||||
// content can be null for thinking/reasoning models or when finish_reason="length".
|
||||
// Fall back to empty string rather than erroring — the partial response is still
|
||||
// useful and a hard error breaks the session.
|
||||
let content = match message["content"].as_str() {
|
||||
Some(s) => s.to_string(),
|
||||
None => {
|
||||
tracing::warn!(
|
||||
finish_reason = finish,
|
||||
?input_tokens,
|
||||
?output_tokens,
|
||||
raw_message = %message,
|
||||
"OpenAI response has null content",
|
||||
);
|
||||
String::new()
|
||||
}
|
||||
};
|
||||
let truncated = finish == "length";
|
||||
LlmTurn::Message(ChatResponse { content, input_tokens, output_tokens, truncated, reasoning_content, cache_read_tokens, cache_creation_tokens: None, cost })
|
||||
};
|
||||
|
||||
Ok((turn, Some(raw_meta)))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user