feat(read_file): let capable models view images, video and PDFs
Nightly Build / build (push) Successful in 6m38s
Nightly Build / build (push) Successful in 6m38s
When the resolved model declares an input modality (vision → images,
video, document → PDFs), read_file now hands a binary media file back to
the model as native input instead of failing on non-UTF-8 bytes.
- ToolResult gains a Media { text, media } variant carrying MediaRef
{ host_path, mime }; the tool message keeps only the text note, the
bytes travel out of band in a new additive chat_llm_tools.media column
(mirrors preview_old/new).
- read_file sniffs the resolved host file; a recognized medium becomes a
Media result (with a neutral note), everything else keeps the textual
path. Capability gating lives in the message builder, so read_file
never needs the model caps and degrades cleanly on a text-only model.
- MessageBuilder inlines current-turn tool media as a synthetic user
message right after the tool-result group (media_turn_start boundary,
so older turns are never re-billed), reusing media.rs primitives via a
new inline_paths helper that contains against the caller's workspace
roots. OpenAI forwards the parts verbatim; the Anthropic client now
also translates the PDF `file` part into a native `document` block
(image_url → image was already handled).
- read_file's description is annotated per serving model in
call_llm_round, listing the formats it can open, so the model knows
reading one shows it the content.
Tests: media sniff (PDF), PDF file-part build, inline_paths containment
+ capability gating, capability hint, read_file media-vs-text, Anthropic
file→document, and the owner-schema-stands-alone check with the new
column.
This commit is contained in:
@@ -18,6 +18,10 @@ pub struct LlmToolCall {
|
||||
/// size cap (no diff shown then). Only populated by `for_message` (history).
|
||||
pub preview_old: Option<String>,
|
||||
pub preview_new: Option<String>,
|
||||
/// JSON `[{host_path, mime}]` — media files this tool produced (e.g. `read_file`
|
||||
/// on an image), to be inlined to the model as native input by the message
|
||||
/// builder. `None` for non-media tools. Only populated by `for_message`/`get`.
|
||||
pub media: Option<String>,
|
||||
}
|
||||
|
||||
/// Inserts a tool call in `running` state and returns its id.
|
||||
@@ -81,6 +85,19 @@ pub async fn set_preview(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persists the JSON media manifest for a tool that produced viewable media
|
||||
/// (`ToolResult::Media`). Separate from [`complete`] — like [`set_preview`] — so the
|
||||
/// out-of-band media write stays independent of the status/result write. Read back
|
||||
/// by `for_message` so the message builder can inline the files for the model.
|
||||
pub async fn set_media(pool: &SqlitePool, id: i64, media_json: &str) -> anyhow::Result<()> {
|
||||
sqlx::query("UPDATE chat_llm_tools SET media = ? WHERE id = ?")
|
||||
.bind(media_json)
|
||||
.bind(id)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn fail(pool: &SqlitePool, id: i64, error: &str) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
"UPDATE chat_llm_tools SET result = ?, status = 'failed' WHERE id = ?",
|
||||
@@ -150,8 +167,8 @@ pub async fn for_message(
|
||||
pool: &SqlitePool,
|
||||
message_id: i64,
|
||||
) -> anyhow::Result<Vec<LlmToolCall>> {
|
||||
let rows = sqlx::query_as::<_, (i64, i64, String, Option<String>, Option<String>, String, String, Option<String>, Option<String>)>(
|
||||
"SELECT id, message_id, name, arguments, result, result_type, status, preview_old, preview_new
|
||||
let rows = sqlx::query_as::<_, (i64, i64, String, Option<String>, Option<String>, String, String, Option<String>, Option<String>, Option<String>)>(
|
||||
"SELECT id, message_id, name, arguments, result, result_type, status, preview_old, preview_new, media
|
||||
FROM chat_llm_tools
|
||||
WHERE message_id = ?
|
||||
ORDER BY id ASC",
|
||||
@@ -161,8 +178,8 @@ pub async fn for_message(
|
||||
.await?;
|
||||
|
||||
Ok(rows.into_iter()
|
||||
.map(|(id, message_id, name, arguments, result, result_type, status, preview_old, preview_new)| {
|
||||
LlmToolCall { id, message_id, name, arguments, result, result_type, status, preview_old, preview_new }
|
||||
.map(|(id, message_id, name, arguments, result, result_type, status, preview_old, preview_new, media)| {
|
||||
LlmToolCall { id, message_id, name, arguments, result, result_type, status, preview_old, preview_new, media }
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@@ -170,8 +187,8 @@ pub async fn for_message(
|
||||
/// A single tool call by id, with its diff-preview columns. Backs the tool-detail
|
||||
/// page (`GET /api/tools/{id}`). Returns `None` when the id is unknown in this pool.
|
||||
pub async fn get(pool: &SqlitePool, id: i64) -> anyhow::Result<Option<LlmToolCall>> {
|
||||
let row = sqlx::query_as::<_, (i64, i64, String, Option<String>, Option<String>, String, String, Option<String>, Option<String>)>(
|
||||
"SELECT id, message_id, name, arguments, result, result_type, status, preview_old, preview_new
|
||||
let row = sqlx::query_as::<_, (i64, i64, String, Option<String>, Option<String>, String, String, Option<String>, Option<String>, Option<String>)>(
|
||||
"SELECT id, message_id, name, arguments, result, result_type, status, preview_old, preview_new, media
|
||||
FROM chat_llm_tools
|
||||
WHERE id = ?",
|
||||
)
|
||||
@@ -179,8 +196,8 @@ pub async fn get(pool: &SqlitePool, id: i64) -> anyhow::Result<Option<LlmToolCal
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
|
||||
Ok(row.map(|(id, message_id, name, arguments, result, result_type, status, preview_old, preview_new)| {
|
||||
LlmToolCall { id, message_id, name, arguments, result, result_type, status, preview_old, preview_new }
|
||||
Ok(row.map(|(id, message_id, name, arguments, result, result_type, status, preview_old, preview_new, media)| {
|
||||
LlmToolCall { id, message_id, name, arguments, result, result_type, status, preview_old, preview_new, media }
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -189,6 +206,6 @@ fn row_to_tool(
|
||||
i64, i64, String, Option<String>, Option<String>, String, String,
|
||||
),
|
||||
) -> LlmToolCall {
|
||||
// The resume path (`pending_for_stack`) never needs the diff preview.
|
||||
LlmToolCall { id, message_id, name, arguments, result, result_type, status, preview_old: None, preview_new: None }
|
||||
// The resume path (`pending_for_stack`) never needs the diff preview or media.
|
||||
LlmToolCall { id, message_id, name, arguments, result, result_type, status, preview_old: None, preview_new: None, media: None }
|
||||
}
|
||||
|
||||
@@ -750,14 +750,16 @@ pub async fn create_owner_tables(pool: &SqlitePool) -> Result<()> {
|
||||
result_type TEXT NOT NULL DEFAULT 'string' CHECK(result_type IN ('string', 'json')),
|
||||
preview_old TEXT, -- file-write diff: content before the write
|
||||
preview_new TEXT, -- file-write diff: content after the write
|
||||
media TEXT, -- JSON [{host_path,mime}]: media the tool produced, inlined to the model out of band
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)",
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
// Diff-preview columns are additive — reach an already-created table in place.
|
||||
// Diff-preview + tool-media columns are additive — reach an already-created table in place.
|
||||
ensure_column(pool, "chat_llm_tools", "preview_old", "TEXT").await?;
|
||||
ensure_column(pool, "chat_llm_tools", "preview_new", "TEXT").await?;
|
||||
ensure_column(pool, "chat_llm_tools", "media", "TEXT").await?;
|
||||
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_stack_session ON chat_sessions_stack(session_id)",
|
||||
|
||||
Reference in New Issue
Block a user