feat(dashboard): LLM stats answer who/what spends tokens — member scope chips and spend breakdowns
Nightly Build / build (push) Successful in 4m27s
Nightly Build / build (push) Successful in 4m27s
The stats section was designed single-user: four global charts, no attribution. Request metadata rows now carry the session's source and the frame's agent_id/depth (additive ensure_column), denormalized at log time by the LoggingModel from the owner's pool — off the turn's hot path, degrading to NULLs, never to a lost row. The dashboard gains member scope chips filtering every chart, and a breakdown row splitting the range's billed tokens by member, kind (chat / sub-agents / cron / system agents / channels), agent, model and provider — a sub-agent's spend is attributed to the sub-agent itself. Rows predating the columns group under 'older data'.
This commit is contained in:
@@ -6,7 +6,8 @@
|
||||
//! of the traffic is known — `user_id` is what the UI filters on).
|
||||
//! Payloads (request/response bodies + headers) live in `llm_request_payloads`
|
||||
//! in the owner bucket (`{userid}.db`), correlated by `request_id`.
|
||||
//! Rows are retained for `llm.request_log.retention_days` days (default 14).
|
||||
//! Rows are retained for `llm.requests_log.cleanup_rows_after` days (the
|
||||
//! shipped `default.config.yaml` sets 90; unset = kept forever).
|
||||
|
||||
use anyhow::Result;
|
||||
use sqlx::SqlitePool;
|
||||
@@ -31,6 +32,13 @@ pub struct LlmRequestRow {
|
||||
pub cache_read_tokens: Option<i64>,
|
||||
/// Tokens written into the provider's prompt cache (Anthropic only).
|
||||
pub cache_creation_tokens: Option<i64>,
|
||||
/// Denormalized attribution (see `db::mod`): the session's `source`, the
|
||||
/// frame's `agent_id` (the sub-agent's own for a child frame) and `depth`
|
||||
/// (0 = main agent, >0 = sub-agent). Resolved by the LoggingModel from the
|
||||
/// owner's pool; `None` when it was unavailable.
|
||||
pub source: Option<String>,
|
||||
pub agent_id: Option<String>,
|
||||
pub depth: Option<i64>,
|
||||
}
|
||||
|
||||
// ── Writes ────────────────────────────────────────────────────────────────────
|
||||
@@ -40,8 +48,9 @@ pub async fn insert(pool: &SqlitePool, row: LlmRequestRow) -> Result<i64> {
|
||||
"INSERT INTO llm_requests (
|
||||
request_id, user_id, session_id, stack_id, model_name,
|
||||
error_text, input_tokens, output_tokens, duration_ms,
|
||||
cache_read_tokens, cache_creation_tokens
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
cache_read_tokens, cache_creation_tokens,
|
||||
source, agent_id, depth
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(&row.request_id)
|
||||
@@ -55,6 +64,9 @@ pub async fn insert(pool: &SqlitePool, row: LlmRequestRow) -> Result<i64> {
|
||||
.bind(row.duration_ms)
|
||||
.bind(row.cache_read_tokens)
|
||||
.bind(row.cache_creation_tokens)
|
||||
.bind(&row.source)
|
||||
.bind(&row.agent_id)
|
||||
.bind(row.depth)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -421,6 +421,16 @@ pub(crate) async fn create_registry_tables(pool: &SqlitePool) -> Result<()> {
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
// Attribution columns for the multi-user stats (who/what consumed): the
|
||||
// session's `source`, the frame's `agent_id` and its `depth` (0 = the
|
||||
// conversation's main agent, >0 = a sub-agent). They live in the owner
|
||||
// bucket, so the LoggingModel denormalizes them onto the row at insert
|
||||
// time — a registry query cannot join an encrypted per-user file. NULL on
|
||||
// rows predating the columns and when the owner's pool was not available.
|
||||
ensure_column(pool, "llm_requests", "source", "TEXT").await?;
|
||||
ensure_column(pool, "llm_requests", "agent_id", "TEXT").await?;
|
||||
ensure_column(pool, "llm_requests", "depth", "INTEGER").await?;
|
||||
|
||||
// User directory + auth material. Read before every login, so it lives in
|
||||
// the registry — which means it must never hold anything that derives a
|
||||
// user's key: `database_password` is the DEK sealed under a key derived
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
//!
|
||||
//! * a **metadata-only** row in `llm_requests` (`system.db`) — cost, tokens,
|
||||
//! timing, plus the correlation the UI filters on (`user_id`, `session_id`,
|
||||
//! `stack_id`);
|
||||
//! `stack_id`) and the denormalized attribution the dashboard breaks down on
|
||||
//! (`source`, `agent_id`, `depth`);
|
||||
//! * the **payload** (request/response bodies + headers) in
|
||||
//! `llm_request_payloads` in the caller's own database, keyed by the same
|
||||
//! `request_id`.
|
||||
@@ -90,6 +91,90 @@ impl LoggingModel {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the denormalized attribution columns from the owner's pool: the
|
||||
/// session's `source`, and the frame's `agent_id`/`depth`. The frame row
|
||||
/// carries the sub-agent's own id for a child frame, so it wins over the
|
||||
/// session's agent; the root frame carries the session's main agent. Runs
|
||||
/// inside the spawned insert task — never on the turn's hot path — and any
|
||||
/// failure degrades to NULLs, never to a lost metadata row.
|
||||
async fn resolve_attribution(
|
||||
pool: &SqlitePool,
|
||||
session_id: i64,
|
||||
stack_id: Option<i64>,
|
||||
) -> (Option<String>, Option<String>, Option<i64>) {
|
||||
let session = sqlx::query_as::<_, (String, String)>(
|
||||
"SELECT source, agent_id FROM chat_sessions WHERE id = ?",
|
||||
)
|
||||
.bind(session_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
|
||||
let frame = match stack_id {
|
||||
Some(id) => sqlx::query_as::<_, (i64, String)>(
|
||||
"SELECT depth, agent_id FROM chat_sessions_stack WHERE id = ?",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten(),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let source = session.as_ref().map(|(s, _)| s.clone());
|
||||
let agent_id = frame
|
||||
.as_ref()
|
||||
.map(|(_, a)| a.clone())
|
||||
.or_else(|| session.as_ref().map(|(_, a)| a.clone()));
|
||||
let depth = frame.as_ref().map(|(d, _)| *d);
|
||||
(source, agent_id, depth)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn spawn_insert(
|
||||
registry: Arc<SqlitePool>,
|
||||
owner: Option<Arc<SqlitePool>>,
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
session_id: Option<i64>,
|
||||
stack_id: Option<i64>,
|
||||
model_name: String,
|
||||
error_text: Option<String>,
|
||||
duration_ms: i64,
|
||||
// (input, output, cache_read, cache_creation) — all None on HTTP failure.
|
||||
usage: (Option<i64>, Option<i64>, Option<i64>, Option<i64>),
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
// Attribution needs the owner's pool (chat_sessions lives there); a
|
||||
// locked owner still gets the metadata row, only without attribution.
|
||||
let (source, agent_id, depth) = match (owner.as_deref(), session_id) {
|
||||
(Some(p), Some(sid)) => resolve_attribution(p, sid, stack_id).await,
|
||||
_ => (None, None, None),
|
||||
};
|
||||
let (input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens) = usage;
|
||||
if let Err(e) = llm_requests::insert(®istry, llm_requests::LlmRequestRow {
|
||||
request_id,
|
||||
user_id,
|
||||
session_id,
|
||||
stack_id,
|
||||
model_name,
|
||||
error_text,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
duration_ms,
|
||||
cache_read_tokens,
|
||||
cache_creation_tokens,
|
||||
source,
|
||||
agent_id,
|
||||
depth,
|
||||
}).await {
|
||||
warn!(error = %e, "llm_requests: failed to insert log row");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Model for LoggingModel {
|
||||
async fn complete(
|
||||
@@ -108,11 +193,12 @@ impl Model for LoggingModel {
|
||||
let request_id = Some(req.request_id.clone());
|
||||
let model_name = self.model_name.clone();
|
||||
let pool = Arc::clone(&self.registry);
|
||||
let owner = self.target.payloads.clone();
|
||||
|
||||
match &result {
|
||||
Ok(resp) => {
|
||||
let usage = resp.usage();
|
||||
let (input_tokens, output_tokens, cache_read, cache_write) = (
|
||||
let usage = (
|
||||
usage.input_tokens.map(|n| n as i64),
|
||||
usage.output_tokens.map(|n| n as i64),
|
||||
usage.cache_read.map(|n| n as i64),
|
||||
@@ -121,23 +207,8 @@ impl Model for LoggingModel {
|
||||
if let Some(raw) = resp.raw() {
|
||||
self.spawn_payload(&req.request_id, raw);
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = llm_requests::insert(&pool, llm_requests::LlmRequestRow {
|
||||
request_id,
|
||||
user_id,
|
||||
session_id,
|
||||
stack_id,
|
||||
model_name,
|
||||
error_text: None,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
duration_ms,
|
||||
cache_read_tokens: cache_read,
|
||||
cache_creation_tokens: cache_write,
|
||||
}).await {
|
||||
warn!(error = %e, "llm_requests: failed to insert log row");
|
||||
}
|
||||
});
|
||||
spawn_insert(pool, owner, request_id, user_id, session_id, stack_id,
|
||||
model_name, None, duration_ms, usage);
|
||||
}
|
||||
Err(e) => {
|
||||
// Only an HTTP failure carries a body (a provider 400 is exactly
|
||||
@@ -145,24 +216,9 @@ impl Model for LoggingModel {
|
||||
if let Some(raw) = e.raw.as_ref() {
|
||||
self.spawn_payload(&req.request_id, raw);
|
||||
}
|
||||
let error_text = e.to_string();
|
||||
tokio::spawn(async move {
|
||||
if let Err(log_err) = llm_requests::insert(&pool, llm_requests::LlmRequestRow {
|
||||
request_id,
|
||||
user_id,
|
||||
session_id,
|
||||
stack_id,
|
||||
model_name,
|
||||
error_text: Some(error_text),
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
duration_ms,
|
||||
cache_read_tokens: None,
|
||||
cache_creation_tokens: None,
|
||||
}).await {
|
||||
warn!(error = %log_err, "llm_requests: failed to insert error log row");
|
||||
}
|
||||
});
|
||||
spawn_insert(pool, owner, request_id, user_id, session_id, stack_id,
|
||||
model_name, Some(e.to_string()), duration_ms,
|
||||
(None, None, None, None));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,6 +296,14 @@ mod tests {
|
||||
let path = temp_db_path("llmlog-ok");
|
||||
let pool = Arc::new(crate::db::init_system_pool(&path).await.unwrap());
|
||||
|
||||
// The attribution the row should carry: session 42 is a cron session
|
||||
// whose agent is "assistant", frame 7 is a child frame of the
|
||||
// "researcher" sub-agent — the frame's agent wins over the session's.
|
||||
sqlx::query("INSERT INTO chat_sessions (id, source, agent_id) VALUES (42, 'cron', 'assistant')")
|
||||
.execute(&*pool).await.unwrap();
|
||||
sqlx::query("INSERT INTO chat_sessions_stack (id, session_id, agent_id, depth) VALUES (7, 42, 'researcher', 1)")
|
||||
.execute(&*pool).await.unwrap();
|
||||
|
||||
let mut resp = ModelResponse::message("hi");
|
||||
*resp.usage_mut() = Usage {
|
||||
input_tokens: Some(11),
|
||||
@@ -270,6 +334,13 @@ mod tests {
|
||||
assert_eq!(model_name, "gpt-test");
|
||||
assert_eq!((input, output), (Some(11), Some(7)));
|
||||
|
||||
let (source, agent_id, depth): (Option<String>, Option<String>, Option<i64>) =
|
||||
sqlx::query_as("SELECT source, agent_id, depth FROM llm_requests WHERE request_id = 'req-1'")
|
||||
.fetch_one(&*pool).await.unwrap();
|
||||
assert_eq!(source.as_deref(), Some("cron"), "the dashboard breaks down on source");
|
||||
assert_eq!(agent_id.as_deref(), Some("researcher"), "the frame's agent wins (sub-agent)");
|
||||
assert_eq!(depth, Some(1));
|
||||
|
||||
wait_for(&pool, "SELECT COUNT(*) FROM llm_request_payloads").await;
|
||||
let body: String = sqlx::query_scalar(
|
||||
"SELECT request_json FROM llm_request_payloads WHERE request_id = 'req-1'")
|
||||
|
||||
Reference in New Issue
Block a user