token streaming & reasoning display: live SSE tokens frontend to back
Nightly Build / build (push) Successful in 6m59s

- Add StreamDelta(SseDecoder) framing shared by OpenAI/Anthropic
- OpenAiClient: stream=true + reasoning_content deltas, index-based
  tool_calls accumulation, usage from final chunk
- AnthropicClient: message_start/content_block_*/message_delta events,
  thinking_delta->reasoning, input_json_delta->tool input
- TokenDelta ServerEvent variant wired through ChatHub + WS broadcast
- Frontend throttled flush (~15 Hz), pending bubble mutate-in-place,
  reasoning as collapsed-by-default <details>
- Drop streaming bubble on error/llm_failed/model_fallback
- i18n: chat.reasoning key added to en/fr/it
This commit is contained in:
2026-07-22 12:59:13 +01:00
parent e1d285e7db
commit 3343260bb0
23 changed files with 950 additions and 113 deletions
+2 -2
View File
@@ -2,6 +2,6 @@ pub mod logging;
// Re-export from the independent llm-client crate.
pub use llm_client::{
ChatOptions, ChatResponse, ChatbotClient, LlmError, LlmRawMeta, LlmTurn, Message, ToolCall,
anthropic, http_status, lm_studio, ollama, openai,
ChatOptions, ChatResponse, ChatbotClient, LlmError, LlmRawMeta, LlmTurn, Message, StreamDelta,
ToolCall, anthropic, http_status, lm_studio, ollama, openai,
};
+1 -1
View File
@@ -1,3 +1,3 @@
pub use core_api::events::{
ClientMessage, GlobalEvent, InboundDataMessage, ServerEvent,
ClientMessage, GlobalEvent, InboundDataMessage, ServerEvent, TokenDeltaKind,
};
@@ -42,13 +42,19 @@ impl<'a> TurnEmitter<'a> {
}
/// The assistant produced text alongside tool calls (reasoning before acting).
pub(super) async fn thinking(&self, message_id: i64, content: String, input_tokens: Option<u32>, output_tokens: Option<u32>) {
self.emit(ServerEvent::Thinking { message_id, content, input_tokens, output_tokens }).await;
pub(super) async fn thinking(&self, message_id: i64, content: String, input_tokens: Option<u32>, output_tokens: Option<u32>, reasoning_content: Option<String>) {
self.emit(ServerEvent::Thinking { message_id, content, input_tokens, output_tokens, reasoning_content }).await;
}
/// Clone of the underlying sender, for spawning side-channel tasks that
/// emit alongside the turn (e.g. the token-delta forwarder).
pub(super) fn sender(&self) -> mpsc::Sender<ServerEvent> {
self.tx.clone()
}
/// The assistant response is complete.
pub(super) async fn done(&self, message_id: i64, stack_id: i64, content: String, input_tokens: Option<u32>, output_tokens: Option<u32>) {
self.emit(ServerEvent::Done { message_id, stack_id, content, input_tokens, output_tokens }).await;
pub(super) async fn done(&self, message_id: i64, stack_id: i64, content: String, input_tokens: Option<u32>, output_tokens: Option<u32>, reasoning_content: Option<String>) {
self.emit(ServerEvent::Done { message_id, stack_id, content, input_tokens, output_tokens, reasoning_content }).await;
}
/// The LLM was cut off by the token limit.
@@ -9,11 +9,13 @@ use std::collections::HashSet;
use std::sync::Arc;
use serde_json::Value;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{error, warn};
use crate::chatbot::{ChatOptions, LlmTurn};
use crate::chatbot::{ChatOptions, LlmTurn, StreamDelta};
use crate::db::llm_request_payloads;
use crate::events::{ServerEvent, TokenDeltaKind};
use crate::llm::{LlmEntry, LlmStrength};
use super::ChatSessionHandler;
@@ -77,10 +79,19 @@ impl ChatSessionHandler {
// the fallback reassignment below. On cancel we drop the future
// (aborting the request) and return immediately.
let client = cur_llm.client.clone();
// Streaming side-channel: providers that support SSE push deltas here;
// the forwarder re-emits them as `TokenDelta` events on the turn bus.
// Best-effort — the round's final events remain authoritative.
let (delta_tx, delta_rx) = mpsc::channel::<StreamDelta>(256);
let forwarder = spawn_delta_forwarder(delta_rx, em.sender());
let call_result = tokio::select! {
_ = token.cancelled() => return RoundLlm::Cancelled,
r = client.chat_with_tools_raw(messages.as_slice(), defs, &options) => r,
r = client.chat_with_tools_raw_streaming(messages.as_slice(), defs, &options, delta_tx) => r,
};
// The client's sender dropped with the completed future: the forwarder
// drains any queued deltas and exits, so every `TokenDelta` precedes the
// round's outcome events (Thinking / Done) in bus order.
forwarder.await.ok();
let e = match call_result {
Ok((turn, meta)) => {
@@ -151,6 +162,26 @@ impl ChatSessionHandler {
}
}
/// Forwards streaming deltas from the LLM client onto the turn's event channel
/// as `TokenDelta` events. Exits when the client drops its sender (call
/// completed or aborted) or when the turn receiver is gone.
fn spawn_delta_forwarder(
mut rx: mpsc::Receiver<StreamDelta>,
tx: mpsc::Sender<ServerEvent>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(d) = rx.recv().await {
let (kind, delta) = match d {
StreamDelta::Text(t) => (TokenDeltaKind::Content, t),
StreamDelta::Reasoning(t) => (TokenDeltaKind::Reasoning, t),
};
if tx.send(ServerEvent::TokenDelta { kind, delta }).await.is_err() {
break;
}
}
})
}
/// Whether an LLM error is worth retrying on a different model.
///
/// Classifies on the real HTTP status ([`crate::chatbot::http_status`]), not a
@@ -152,6 +152,7 @@ impl ChatSessionHandler {
input_tokens: resp.input_tokens,
output_tokens: resp.output_tokens,
truncated: resp.truncated,
reasoning_content: resp.reasoning_content,
tool_calls: all_tool_calls,
});
}
@@ -165,7 +166,7 @@ impl ChatSessionHandler {
chat_history::set_usage(pool, message_id, i, o, 0, cost).await?;
}
if !assistant_text.trim().is_empty() || input_tokens.is_some() {
em.thinking(message_id, assistant_text, input_tokens, output_tokens).await;
em.thinking(message_id, assistant_text, input_tokens, output_tokens, reasoning_content).await;
}
// A homogeneous batch of ≥2 synchronous sub-agent calls is fanned
+4 -2
View File
@@ -103,6 +103,8 @@ pub(super) enum TurnOutcome {
input_tokens: Option<u32>,
output_tokens: Option<u32>,
truncated: bool,
/// Chain-of-thought produced by the final round, when any.
reasoning_content: Option<String>,
/// All tool calls executed during this turn, across all rounds.
tool_calls: Vec<crate::chat_event_bus::ToolCallEvent>,
},
@@ -643,7 +645,7 @@ impl ChatSessionHandler {
let outcome = self.run_agent_turn(stack.id, &config, &token, &tx, pending_input.as_ref()).await?;
match outcome {
TurnOutcome::Final { content, message_id, input_tokens, output_tokens, truncated, tool_calls } => {
TurnOutcome::Final { content, message_id, input_tokens, output_tokens, truncated, reasoning_content, tool_calls } => {
// Persist token count so the *next* handle_message call knows
// whether to compact before running the LLM loop.
if let Some(t) = input_tokens {
@@ -654,7 +656,7 @@ impl ChatSessionHandler {
warn!(session_id = self.session_id, ?output_tokens, "response truncated (max_tokens)");
em.truncated(output_tokens).await;
}
em.done(message_id, stack.id, content.clone(), input_tokens, output_tokens).await;
em.done(message_id, stack.id, content.clone(), input_tokens, output_tokens, reasoning_content).await;
// Publish both messages to the event bus now that both are in the DB.
let now = chrono::Utc::now();
@@ -128,6 +128,7 @@ impl ChatSessionHandler {
input_tokens: None,
output_tokens: None,
truncated: false,
reasoning_content: msg.reasoning_content,
tool_calls: Vec::new(),
};
break 'seed (outcome, stack);
@@ -208,13 +209,13 @@ impl ChatSessionHandler {
// current_stack is now the root (depth=0); emit the final event.
match current_outcome {
TurnOutcome::Final { content, message_id, input_tokens, output_tokens, truncated, .. } => {
TurnOutcome::Final { content, message_id, input_tokens, output_tokens, truncated, reasoning_content, .. } => {
info!(session_id = self.session_id, "resume_turn done");
if truncated {
warn!(session_id = self.session_id, "response truncated");
em.truncated(output_tokens).await;
}
em.done(message_id, current_stack.id, content, input_tokens, output_tokens).await;
em.done(message_id, current_stack.id, content, input_tokens, output_tokens, reasoning_content).await;
}
TurnOutcome::Cancelled => {
info!(session_id = self.session_id, "resume_turn cancelled");