llm: switch Skald to agent-loop Model clients; drop llm-client (phase 1, D13)
The LLM call path now runs on the agent-loop crate's clients and trait: - core-api: BuiltLlmClient.client is Arc<dyn agent_loop::model::Model>; chatbot.rs (ChatbotClient + wire types) deleted; APP_NAME re-exported from agent-loop - providers (openai/anthropic/ollama/openrouter/requesty/declared) build OpenAiModel/AnthropicModel/OllamaModel with the model's wire id - LoggingModel decorator (llm/logging.rs) replaces LoggingChatbotClient; per-request correlation (session/stack/user) travels in the new ModelRequest.log field, never sent to providers - llm_call/llm_loop/compactor speak Model::complete + ModelResponse; retriability via Model::is_retriable (structured status, B6 rule now the crate's default); payload persistence reads RawMeta off ModelResponse/ModelError - crates/llm-client and skald-core/src/chatbot deleted Full workspace test suite green (incl. 162 skald-core + 32 agent-loop).
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
//! Transparent logging decorator for any [`agent_loop::model::Model`].
|
||||
//!
|
||||
//! [`LoggingModel`] intercepts every `complete` call, measures the duration,
|
||||
//! and persists a **metadata-only** row to `llm_requests` in `system.db`
|
||||
//! (fire-and-forget). Per-request correlation (session/stack/user id) travels
|
||||
//! in [`ModelRequest::log`], set by the caller; the payload (request/response
|
||||
//! bodies) is returned to the caller inside [`ModelResponse::raw`] /
|
||||
//! [`ModelError::raw`] so it can be written to the user's own database.
|
||||
//!
|
||||
//! The split keeps conversation content (payloads) behind the user key while
|
||||
//! metadata (cost, tokens, timing) stays in the admin-readable registry.
|
||||
//! (Successor of `chatbot::logging::LoggingChatbotClient`, blueprint D13.)
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sqlx::SqlitePool;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::warn;
|
||||
|
||||
use agent_loop::model::{Model, ModelError, ModelRequest, ModelResponse, StreamDelta};
|
||||
|
||||
use crate::db::llm_requests;
|
||||
|
||||
pub struct LoggingModel {
|
||||
inner: Arc<dyn Model>,
|
||||
pool: Arc<SqlitePool>,
|
||||
model_name: String,
|
||||
}
|
||||
|
||||
impl LoggingModel {
|
||||
pub fn new(inner: Arc<dyn Model>, pool: Arc<SqlitePool>, model_name: impl Into<String>) -> Self {
|
||||
Self { inner, pool, model_name: model_name.into() }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Model for LoggingModel {
|
||||
async fn complete(
|
||||
&self,
|
||||
req: &ModelRequest,
|
||||
deltas: Option<mpsc::Sender<StreamDelta>>,
|
||||
) -> Result<ModelResponse, ModelError> {
|
||||
let start = Instant::now();
|
||||
let result = self.inner.complete(req, deltas).await;
|
||||
let duration_ms = start.elapsed().as_millis() as i64;
|
||||
|
||||
// Per-request correlation set by the caller (llm_call / compactor).
|
||||
let log = req.log.clone().unwrap_or_default();
|
||||
let session_id = log["session_id"].as_i64();
|
||||
let stack_id = log["stack_id"].as_i64();
|
||||
let user_id = log["user_id"].as_str().map(str::to_string);
|
||||
let request_id = Some(req.request_id.clone());
|
||||
let model_name = self.model_name.clone();
|
||||
let pool = Arc::clone(&self.pool);
|
||||
|
||||
match &result {
|
||||
Ok(resp) => {
|
||||
let usage = resp.usage();
|
||||
let (input_tokens, output_tokens, cache_read, cache_write) = (
|
||||
usage.input_tokens.map(|n| n as i64),
|
||||
usage.output_tokens.map(|n| n as i64),
|
||||
usage.cache_read.map(|n| n as i64),
|
||||
usage.cache_write.map(|n| n as i64),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = llm_requests::insert(&pool, llm_requests::LlmRequestRow {
|
||||
request_id,
|
||||
user_id,
|
||||
session_id,
|
||||
stack_id,
|
||||
model_name,
|
||||
error_text: None,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
duration_ms,
|
||||
cache_read_tokens: cache_read,
|
||||
cache_creation_tokens: cache_write,
|
||||
}).await {
|
||||
warn!(error = %e, "llm_requests: failed to insert log row");
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
let error_text = e.to_string();
|
||||
tokio::spawn(async move {
|
||||
if let Err(log_err) = llm_requests::insert(&pool, llm_requests::LlmRequestRow {
|
||||
request_id,
|
||||
user_id,
|
||||
session_id,
|
||||
stack_id,
|
||||
model_name,
|
||||
error_text: Some(error_text),
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
duration_ms,
|
||||
cache_read_tokens: None,
|
||||
cache_creation_tokens: None,
|
||||
}).await {
|
||||
warn!(error = %log_err, "llm_requests: failed to insert error log row");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn is_retriable(&self, err: &ModelError) -> bool {
|
||||
self.inner.is_retriable(err)
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,11 @@ use sqlx::SqlitePool;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::chatbot::ChatbotClient;
|
||||
use crate::chatbot::logging::LoggingChatbotClient;
|
||||
use agent_loop::model::Model;
|
||||
use core_api::provider::LlmStrength;
|
||||
use crate::provider::{ApiProvider, ProviderRegistry, ReasoningMode};
|
||||
|
||||
use super::logging::LoggingModel;
|
||||
use super::providers::RemoteLlmModelInfo;
|
||||
use super::{ClientStatus, LlmEntry, LlmModelInfo, LlmModelRecord, LlmProviderInfo, LlmProviderRecord};
|
||||
use super::db;
|
||||
@@ -512,8 +512,8 @@ fn build_entry(
|
||||
let prompt_cache = built.prompt_cache;
|
||||
let extra = model.extra_params.clone();
|
||||
|
||||
let client: Arc<dyn ChatbotClient> = match log_pool {
|
||||
Some(pool) => Arc::new(LoggingChatbotClient::new(inner, pool, &model.name)),
|
||||
let client: Arc<dyn Model> = match log_pool {
|
||||
Some(pool) => Arc::new(LoggingModel::new(inner, pool, &model.name)),
|
||||
None => inner,
|
||||
};
|
||||
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
pub(crate) mod db;
|
||||
pub mod logging;
|
||||
pub mod manager;
|
||||
pub mod providers;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::chatbot::ChatbotClient;
|
||||
use agent_loop::model::Model;
|
||||
|
||||
use crate::provider::ServiceType;
|
||||
|
||||
pub use core_api::provider::{LlmProviderRecord, LlmModelRecord, LlmStrength, ReasoningMode};
|
||||
@@ -13,7 +15,7 @@ pub use manager::{LlmManager, sort_models_for_agent};
|
||||
/// A resolved, ready-to-use LLM client with its associated metadata.
|
||||
#[derive(Clone)]
|
||||
pub struct LlmEntry {
|
||||
pub client: Arc<dyn ChatbotClient>,
|
||||
pub client: Arc<dyn Model>,
|
||||
pub model: String,
|
||||
pub model_db_id: i64,
|
||||
pub strength: Option<LlmStrength>,
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
|
||||
use crate::chatbot::anthropic::AnthropicClient;
|
||||
use agent_loop::models::AnthropicModel;
|
||||
use crate::llm::{LlmModelRecord, LlmProviderRecord};
|
||||
use crate::llm::providers::{RemoteLlmModelInfo, extra_with_reasoning};
|
||||
use crate::provider::{ApiProvider, BuiltLlmClient, ProviderField, ProviderUiMeta, ReasoningMode, ServiceType};
|
||||
@@ -111,7 +111,7 @@ impl ApiProvider for AnthropicProvider {
|
||||
// stays uncached, as before.
|
||||
let prompt_cache = model.capabilities.iter().any(|c| c == "tool_search");
|
||||
Ok(BuiltLlmClient {
|
||||
client: Arc::new(AnthropicClient::with_extra_body(key, extra)),
|
||||
client: Arc::new(AnthropicModel::with_extra_body(key, model.model_id.clone(), extra)),
|
||||
prompt_cache,
|
||||
})
|
||||
})())
|
||||
|
||||
@@ -18,7 +18,7 @@ use std::sync::Arc;
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::chatbot::openai::OpenAiClient;
|
||||
use agent_loop::models::OpenAiModel;
|
||||
use crate::llm::providers::{extra_with_reasoning, RemoteLlmModelInfo};
|
||||
use crate::llm::{LlmModelRecord, LlmProviderRecord};
|
||||
use crate::provider::{
|
||||
@@ -575,7 +575,7 @@ impl ApiProvider for DeclaredProvider {
|
||||
let extra = extra_with_reasoning(self, model);
|
||||
let prompt_cache = self.spec.prompt_cache;
|
||||
Ok(BuiltLlmClient {
|
||||
client: Arc::new(OpenAiClient::new(self.base_url(record), key, extra, prompt_cache)),
|
||||
client: Arc::new(OpenAiModel::with_options(self.base_url(record), key, model.model_id.clone(), extra, prompt_cache)),
|
||||
prompt_cache,
|
||||
})
|
||||
})())
|
||||
|
||||
@@ -15,7 +15,7 @@ use anyhow::{anyhow, Context, Result};
|
||||
|
||||
use core_api::provider::{ApiProvider, BuiltLlmClient, LlmModelRecord, LlmProviderRecord};
|
||||
|
||||
use crate::chatbot::openai::OpenAiClient;
|
||||
use agent_loop::models::OpenAiModel;
|
||||
|
||||
/// Computes the `extra_params` an OpenAI-compatible client should be built with,
|
||||
/// given a model's stored `extra_params` and its selected reasoning value. The
|
||||
@@ -75,7 +75,7 @@ pub(crate) async fn fetch_openai_models(
|
||||
.ok_or_else(|| anyhow!("unexpected {who} response shape"))
|
||||
}
|
||||
|
||||
/// Builds an `OpenAiClient` for an OpenAI-compatible provider: requires the
|
||||
/// Builds an `OpenAiModel` for an OpenAI-compatible provider: requires the
|
||||
/// provider record's `api_key` and merges the model's stored `extra_params`
|
||||
/// with the provider-translated reasoning fragment (see `extra_with_reasoning`).
|
||||
pub(crate) fn build_openai_llm(
|
||||
@@ -89,7 +89,7 @@ pub(crate) fn build_openai_llm(
|
||||
.with_context(|| format!("provider '{}': api_key required for {}", record.name, provider.type_id()))?;
|
||||
let extra = extra_with_reasoning(provider, model);
|
||||
Ok(BuiltLlmClient {
|
||||
client: Arc::new(OpenAiClient::new(base_url, key, extra, prompt_cache)),
|
||||
client: Arc::new(OpenAiModel::with_options(base_url, key, model.model_id.clone(), extra, prompt_cache)),
|
||||
prompt_cache,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::sync::Arc;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
|
||||
use crate::chatbot::ollama::OllamaClient;
|
||||
use agent_loop::models::OllamaModel;
|
||||
use crate::llm::{LlmModelRecord, LlmProviderRecord};
|
||||
use crate::llm::providers::RemoteLlmModelInfo;
|
||||
use crate::provider::{ApiProvider, BuiltLlmClient, ProviderField, ProviderUiMeta, ServiceType};
|
||||
@@ -101,9 +101,9 @@ impl ApiProvider for OllamaProvider {
|
||||
Ok(Some(Self::parse_model_info(&resp, model_id)))
|
||||
}
|
||||
|
||||
fn build_llm(&self, record: &LlmProviderRecord, _model: &LlmModelRecord) -> Option<Result<BuiltLlmClient>> {
|
||||
fn build_llm(&self, record: &LlmProviderRecord, model: &LlmModelRecord) -> Option<Result<BuiltLlmClient>> {
|
||||
Some(Ok(BuiltLlmClient {
|
||||
client: Arc::new(OllamaClient::new(record.base_url.as_deref())),
|
||||
client: Arc::new(OllamaModel::new(record.base_url.as_deref(), model.model_id.clone())),
|
||||
prompt_cache: false,
|
||||
}))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user