llm: add dynamic tool loading (DTL) — Kimi system-tools + Anthropic tool-reference
Nightly Build / build (push) Successful in 6m51s
Nightly Build / build (push) Successful in 6m51s
Replace the old session_mcp_grants/stack_mcp_grants table pair with
a single activated_tools table that anchors each activation at the
assistant message_id that triggered it. The durable write moves from
the activate_tools tool itself to the round loop (handle_tool_call),
which has the message_id the DTL serializer positions injected tool
blocks against.
Introduce DtlMode (None / AnthropicToolReference / KimiSystemTools),
resolved per model from capabilities (opt-in via tool_search)
combined with the provider's dtl_format(). The message builder inserts
Kimi system {tools} blocks at the activation position, or emits
Anthropic tool_reference markers on the tool result. The tool-def
surface (all_tool_defs) switches shape: Anthropic declares everything
deferred; Kimi omits activated tools from the top-level array (system
takes over); None keeps the old grant-set logic.
Anthropic client: accept structured system arrays (cache_control on
the static block when DTL is active), carry defer_loading through
conversion, emit tool_reference blocks on result messages. Prompt
caching enabled exactly when DTL is active (anthropic provider).
MCP server list in the prompt is now a static catalogue (not split
Available/Active) — the split invalidated the cache on every activation.
Groundwork for providers.yaml dtl: key; Moonshot/Kimi providers wired
with kimi_system_tools and the k3* enrich now adds tool_search.
Compactor re-anchors activations whose message was compacted away.
This commit is contained in:
@@ -531,6 +531,16 @@ fn build_entry(
|
||||
context_length: model.context_length,
|
||||
prompt_cache,
|
||||
capabilities: model.capabilities.clone(),
|
||||
// DTL is opt-in per model (the `tool_search` capability); the wire *format*
|
||||
// comes from the model's provider (native, or `providers.yaml`) — no
|
||||
// hardcoded model list.
|
||||
dtl: if model.capabilities.iter().any(|c| c == "tool_search") {
|
||||
registry.get(&provider.provider)
|
||||
.and_then(|p| p.dtl_format().map(crate::llm::dtl_mode_from_format))
|
||||
.unwrap_or(crate::llm::DtlMode::None)
|
||||
} else {
|
||||
crate::llm::DtlMode::None
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,50 @@ pub struct LlmEntry {
|
||||
/// Input capabilities of the resolved model (`vision`, `video`, …), from
|
||||
/// `llm_models.capabilities`. Drives multimodal attachment inlining.
|
||||
pub capabilities: Vec<String>,
|
||||
/// Dynamic-tool-loading serialization mode for this model (resolved from
|
||||
/// `capabilities` + provider type). Selects how a session's *activated* tools
|
||||
/// are put on the wire so that activating one does not invalidate the
|
||||
/// provider's prompt-cache prefix.
|
||||
pub dtl: DtlMode,
|
||||
}
|
||||
|
||||
/// Per-model dynamic-tool-loading (DTL) serialization mode. Resolved in
|
||||
/// `build_entry` from the model's provider (via [`dtl_mode_from_format`]) gated by
|
||||
/// the `tool_search` capability. It selects how a session's activated tools are serialized so
|
||||
/// that an `activate_tools` call does not break the provider's prompt-cache
|
||||
/// prefix. The persistence layer (`activated_tools`) is model-agnostic; this is
|
||||
/// the model-aware half that renders that state per provider.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum DtlMode {
|
||||
/// Today's behaviour: activated tools ride in the top-level `tools` array.
|
||||
/// Correct, but every activation invalidates the cache from that point on.
|
||||
/// The fallback for Ollama / LM Studio / generic OpenAI-compat providers.
|
||||
#[default]
|
||||
None,
|
||||
/// Anthropic Messages API: candidate tools are declared `defer_loading:true`
|
||||
/// and loaded via a custom client-side `tool_reference` expansion emitted at
|
||||
/// the `activate_tools` result (preserves the cache; no 5-result cap).
|
||||
AnthropicToolReference,
|
||||
/// Kimi K3 (OpenAI-compatible): activated tools are injected as `system`
|
||||
/// messages carrying a `tools` field, appended at the activation position so
|
||||
/// the prefix stays byte-identical (append-only).
|
||||
KimiSystemTools,
|
||||
}
|
||||
|
||||
/// Parses a provider-declared DTL format name — from a native provider
|
||||
/// (`AnthropicProvider::dtl_format`) or from `providers.yaml` (`dtl:` on a declared
|
||||
/// provider) — into a [`DtlMode`]. Unknown names → [`DtlMode::None`].
|
||||
///
|
||||
/// The *format* is a property of the provider (which wire its client speaks);
|
||||
/// whether a given model *uses* it is gated separately by the `tool_search`
|
||||
/// capability (see `build_entry`). So there is no hardcoded model list — enabling
|
||||
/// a new Kimi-compatible provider is a `providers.yaml` edit.
|
||||
pub fn dtl_mode_from_format(fmt: &str) -> DtlMode {
|
||||
match fmt {
|
||||
"anthropic_tool_reference" => DtlMode::AnthropicToolReference,
|
||||
"kimi_system_tools" => DtlMode::KimiSystemTools,
|
||||
_ => DtlMode::None,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Provider ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -25,6 +25,12 @@ impl ApiProvider for AnthropicProvider {
|
||||
&[ServiceType::Llm]
|
||||
}
|
||||
|
||||
fn dtl_format(&self) -> Option<&str> {
|
||||
// Every Anthropic model that opts in (via the `tool_search` capability)
|
||||
// uses the custom client-side tool_reference format.
|
||||
Some("anthropic_tool_reference")
|
||||
}
|
||||
|
||||
async fn list_llm_models(&self, _record: &LlmProviderRecord) -> Result<Option<Vec<RemoteLlmModelInfo>>> {
|
||||
Ok(None)
|
||||
}
|
||||
@@ -97,9 +103,16 @@ impl ApiProvider for AnthropicProvider {
|
||||
.with_context(|| format!("provider '{}': api_key required for anthropic", record.name))?;
|
||||
// Merge model extra_params + reasoning (thinking) into the request body.
|
||||
let extra = extra_with_reasoning(self, model);
|
||||
// Prompt caching is enabled exactly when this model runs dynamic tool
|
||||
// loading (the `tool_search` capability → custom tool_reference): the
|
||||
// deferred toolset keeps the tools prefix stable and the message builder
|
||||
// tags the static system block with cache_control, which the client
|
||||
// renders into the `system` array. Without DTL the native Anthropic path
|
||||
// stays uncached, as before.
|
||||
let prompt_cache = model.capabilities.iter().any(|c| c == "tool_search");
|
||||
Ok(BuiltLlmClient {
|
||||
client: Arc::new(AnthropicClient::with_extra_body(key, extra)),
|
||||
prompt_cache: false,
|
||||
prompt_cache,
|
||||
})
|
||||
})())
|
||||
}
|
||||
|
||||
@@ -59,6 +59,11 @@ struct ProviderSpec {
|
||||
fields: Vec<FieldSpec>,
|
||||
models: Option<ModelsSpec>,
|
||||
reasoning: Option<ReasoningSpec>,
|
||||
/// Dynamic-tool-loading wire format for this provider's models (e.g.
|
||||
/// `kimi_system_tools`). Applied only to a model with the `tool_search`
|
||||
/// capability. Absent → no DTL for this provider.
|
||||
#[serde(default)]
|
||||
dtl: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
@@ -506,6 +511,10 @@ impl ApiProvider for DeclaredProvider {
|
||||
LLM_ONLY
|
||||
}
|
||||
|
||||
fn dtl_format(&self) -> Option<&str> {
|
||||
self.spec.dtl.as_deref()
|
||||
}
|
||||
|
||||
async fn list_llm_models(
|
||||
&self,
|
||||
record: &LlmProviderRecord,
|
||||
|
||||
Reference in New Issue
Block a user