token streaming & reasoning display: live SSE tokens frontend to back
Nightly Build / build (push) Successful in 6m59s

- Add StreamDelta(SseDecoder) framing shared by OpenAI/Anthropic
- OpenAiClient: stream=true + reasoning_content deltas, index-based
  tool_calls accumulation, usage from final chunk
- AnthropicClient: message_start/content_block_*/message_delta events,
  thinking_delta->reasoning, input_json_delta->tool input
- TokenDelta ServerEvent variant wired through ChatHub + WS broadcast
- Frontend throttled flush (~15 Hz), pending bubble mutate-in-place,
  reasoning as collapsed-by-default <details>
- Drop streaming bubble on error/llm_failed/model_fallback
- i18n: chat.reasoning key added to en/fr/it
This commit is contained in:
2026-07-22 12:59:13 +01:00
parent e1d285e7db
commit 3343260bb0
23 changed files with 950 additions and 113 deletions
+30
View File
@@ -1,5 +1,6 @@
use async_trait::async_trait;
use serde_json::Value;
use tokio::sync::mpsc;
/// A single message in a conversation.
#[derive(Debug, Clone)]
@@ -90,6 +91,17 @@ pub struct ToolCall {
pub arguments: Value,
}
/// An incremental piece of a streaming completion, pushed by providers that
/// support SSE streaming. Purely best-effort UI feedback: the final `LlmTurn`
/// remains the authoritative result.
#[derive(Debug, Clone)]
pub enum StreamDelta {
/// Visible answer text.
Text(String),
/// Chain-of-thought / reasoning tokens (thinking models).
Reasoning(String),
}
/// Result of one LLM turn when tools are available.
#[derive(Debug)]
pub enum LlmTurn {
@@ -160,4 +172,22 @@ pub trait ChatbotClient: Send + Sync {
) -> anyhow::Result<(LlmTurn, Option<LlmRawMeta>)> {
self.chat_with_tools(messages, tools, options).await.map(|t| (t, None))
}
/// Like `chat_with_tools_raw`, but the provider may push incremental
/// [`StreamDelta`]s into `delta_tx` as tokens arrive (SSE streaming).
/// Senders should use `try_send` and drop deltas when the channel is full —
/// streaming is best-effort UI feedback and must never backpressure the
/// HTTP read. The returned `LlmTurn` is always the complete, authoritative
/// result. The default ignores the channel and falls back to the buffered
/// call, so providers without streaming behave exactly as before.
async fn chat_with_tools_raw_streaming(
&self,
messages: &[Value],
tools: &[Value],
options: &ChatOptions,
delta_tx: mpsc::Sender<StreamDelta>,
) -> anyhow::Result<(LlmTurn, Option<LlmRawMeta>)> {
let _ = delta_tx;
self.chat_with_tools_raw(messages, tools, options).await
}
}
+26
View File
@@ -34,6 +34,16 @@ pub struct GlobalEvent {
// ── Server → Client ───────────────────────────────────────────────────────────
/// Which token stream a [`ServerEvent::TokenDelta`] belongs to.
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TokenDeltaKind {
/// Visible answer text.
Content,
/// Model chain-of-thought (reasoning/thinking tokens).
Reasoning,
}
#[derive(Clone, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ServerEvent {
@@ -117,6 +127,10 @@ pub enum ServerEvent {
content: String,
input_tokens: Option<u32>,
output_tokens: Option<u32>,
/// Chain-of-thought, when the model produced any. Lets non-streaming
/// providers surface the reasoning block live, not just from history.
#[serde(default, skip_serializing_if = "Option::is_none")]
reasoning_content: Option<String>,
},
/// A fatal error occurred processing the request.
Error {
@@ -132,6 +146,17 @@ pub enum ServerEvent {
content: String,
input_tokens: Option<u32>,
output_tokens: Option<u32>,
/// Chain-of-thought of this tool-call round, when produced.
#[serde(default, skip_serializing_if = "Option::is_none")]
reasoning_content: Option<String>,
},
/// One incremental token while the assistant response (or its reasoning)
/// is being generated. Best-effort: deltas ride the lossy broadcast bus and
/// a lagging client may miss some — the final `Done` is always authoritative
/// and carries the complete content.
TokenDelta {
kind: TokenDeltaKind,
delta: String,
},
/// A write operation requires user approval before executing (shows a diff).
PendingWrite {
@@ -279,6 +304,7 @@ impl ServerEvent {
Self::Done { .. } => "done",
Self::Error { .. } => "error",
Self::Thinking { .. } => "thinking",
Self::TokenDelta { .. } => "token_delta",
Self::PendingWrite { .. } => "pending_write",
Self::ApprovalRequired { .. } => "approval_required",
Self::AgentQuestion { .. } => "agent_question",