feat(read_file): let capable models view images, video and PDFs
Nightly Build / build (push) Successful in 6m38s
Nightly Build / build (push) Successful in 6m38s
When the resolved model declares an input modality (vision → images,
video, document → PDFs), read_file now hands a binary media file back to
the model as native input instead of failing on non-UTF-8 bytes.
- ToolResult gains a Media { text, media } variant carrying MediaRef
{ host_path, mime }; the tool message keeps only the text note, the
bytes travel out of band in a new additive chat_llm_tools.media column
(mirrors preview_old/new).
- read_file sniffs the resolved host file; a recognized medium becomes a
Media result (with a neutral note), everything else keeps the textual
path. Capability gating lives in the message builder, so read_file
never needs the model caps and degrades cleanly on a text-only model.
- MessageBuilder inlines current-turn tool media as a synthetic user
message right after the tool-result group (media_turn_start boundary,
so older turns are never re-billed), reusing media.rs primitives via a
new inline_paths helper that contains against the caller's workspace
roots. OpenAI forwards the parts verbatim; the Anthropic client now
also translates the PDF `file` part into a native `document` block
(image_url → image was already handled).
- read_file's description is annotated per serving model in
call_llm_round, listing the formats it can open, so the model knows
reading one shows it the content.
Tests: media sniff (PDF), PDF file-part build, inline_paths containment
+ capability gating, capability hint, read_file media-vs-text, Anthropic
file→document, and the owner-schema-stands-alone check with the new
column.
This commit is contained in:
@@ -277,7 +277,7 @@ mod tests {
|
||||
|
||||
use core_api::user_fs::UserFs;
|
||||
|
||||
use crate::tools::{ExecutionOutcome, Tool, ToolContext};
|
||||
use crate::tools::{ExecutionOutcome, Tool, ToolContext, ToolResult};
|
||||
|
||||
/// A trivial workspace for the memory-routing tests, which never touch disk.
|
||||
fn test_fs() -> Arc<UserFs> {
|
||||
@@ -551,4 +551,50 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&udir);
|
||||
let _ = std::fs::remove_dir_all(&sdir);
|
||||
}
|
||||
|
||||
/// A physical `read_file` on a binary image hands the file back as
|
||||
/// `ToolResult::Media` (host path + sniffed MIME) instead of failing on the
|
||||
/// non-UTF-8 bytes; a UTF-8 file still reads as line-numbered text.
|
||||
#[tokio::test]
|
||||
async fn read_file_returns_media_for_binary_image() {
|
||||
let (shared, sdir) = store("readmedia-shared").await;
|
||||
let (user, udir) = store("readmedia-user").await;
|
||||
|
||||
let root = std::env::temp_dir().join(format!("skald-readmedia-{}", uuid::Uuid::new_v4()));
|
||||
let home = root.join("homes").join("u1");
|
||||
std::fs::create_dir_all(&home).unwrap();
|
||||
let mut png = b"\x89PNG\r\n\x1a\n".to_vec();
|
||||
png.extend_from_slice(&[0xAA; 64]);
|
||||
std::fs::write(home.join("pic.png"), &png).unwrap();
|
||||
std::fs::write(home.join("note.txt"), "hello\nworld").unwrap();
|
||||
|
||||
let fs = Arc::new(UserFs::new(
|
||||
"u1", home.clone(), "skald-u1", PathBuf::from("/root"), vec![], vec![], None,
|
||||
));
|
||||
let ctx = ToolContext { session_id: 1, user_id: "u1".into(), pool: Arc::clone(&user), fs };
|
||||
let read = ReadFile::new(Arc::clone(&shared));
|
||||
|
||||
// image → Media, carrying the resolved host path + MIME.
|
||||
match read.run_with(&ctx, json!({"path": "~/pic.png"})).wait().await {
|
||||
ExecutionOutcome::Completed(ToolResult::Media { text, media }) => {
|
||||
assert!(text.contains("binary media") && text.contains("image/png"), "{text}");
|
||||
assert_eq!(media.len(), 1);
|
||||
assert_eq!(media[0].mime, "image/png");
|
||||
assert!(media[0].host_path.ends_with("pic.png"), "{}", media[0].host_path);
|
||||
}
|
||||
other => panic!("expected Media, got {other:?}"),
|
||||
}
|
||||
|
||||
// UTF-8 text → ordinary numbered text.
|
||||
match read.run_with(&ctx, json!({"path": "~/note.txt"})).wait().await {
|
||||
ExecutionOutcome::Completed(ToolResult::Text(t)) => {
|
||||
assert!(t.contains("| hello") && t.contains("| world"), "{t}");
|
||||
}
|
||||
other => panic!("expected Text, got {other:?}"),
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let _ = std::fs::remove_dir_all(&udir);
|
||||
let _ = std::fs::remove_dir_all(&sdir);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::SqlitePool;
|
||||
|
||||
use crate::tools::{
|
||||
SimpleExecution, Tool, ToolContext, ToolDescriptionLength, ToolExecution, ToolResult,
|
||||
MediaRef, SimpleExecution, Tool, ToolContext, ToolDescriptionLength, ToolExecution, ToolResult,
|
||||
truncate_label, MAX_LABEL_SHORT, MAX_LABEL_FULL,
|
||||
};
|
||||
use super::{classify_memory, read_to_string, MemScope};
|
||||
@@ -47,6 +47,26 @@ fn number_lines(content: &str, start: usize, end_line: Option<usize>, limit: Opt
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// A short, honest note returned as the `tool` message when `read_file` opens a
|
||||
/// binary medium. The bytes travel out of band (`ToolResult::Media`); this text is
|
||||
/// what the model reads in the tool result itself.
|
||||
fn media_note(agent_path: &str, mime: &str, size: u64) -> String {
|
||||
format!(
|
||||
"[read_file: {agent_path} is binary media ({mime}, {}). It is provided to you directly as model input when the current model supports this format; it cannot be shown as text.]",
|
||||
human_size(size),
|
||||
)
|
||||
}
|
||||
|
||||
/// `1536 → "1.5 KiB"`, `2_100_000 → "2.0 MiB"`.
|
||||
fn human_size(bytes: u64) -> String {
|
||||
const KIB: f64 = 1024.0;
|
||||
const MIB: f64 = 1024.0 * 1024.0;
|
||||
let b = bytes as f64;
|
||||
if b >= MIB { format!("{:.1} MiB", b / MIB) }
|
||||
else if b >= KIB { format!("{:.1} KiB", b / KIB) }
|
||||
else { format!("{bytes} B") }
|
||||
}
|
||||
|
||||
impl Tool for ReadFile {
|
||||
fn name(&self) -> &str { "read_file" }
|
||||
fn display_name(&self) -> &str { "Read File" }
|
||||
@@ -110,15 +130,39 @@ impl Tool for ReadFile {
|
||||
}
|
||||
}
|
||||
|
||||
/// Routes `user-memory/…` / `shared-memory/…` to the note store; every other
|
||||
/// path falls through to the on-disk [`execute`](Self::execute).
|
||||
/// Routes `user-memory/…` / `shared-memory/…` to the note store; a physical
|
||||
/// path resolves to the caller's host workspace and is read there — as native
|
||||
/// media when it sniffs as an image/video/PDF (so a vision/document model can
|
||||
/// see it), otherwise as UTF-8 text with line numbers.
|
||||
fn run_with<'a>(&'a self, ctx: &ToolContext, args: Value) -> Box<dyn ToolExecution + 'a> {
|
||||
let path = super::path_arg(&args).unwrap_or_default();
|
||||
let Some(m) = classify_memory(&path) else {
|
||||
return match super::rewrite_to_host(&ctx.fs, &path, args) {
|
||||
Ok(args) => self.run(args),
|
||||
Err(e) => super::error_exec(e.to_string()),
|
||||
// Physical path: resolve + containment-check up front (so an escape
|
||||
// fails immediately), then read inside the work future.
|
||||
let host = match super::resolve_host_path(&ctx.fs, &path) {
|
||||
Ok(h) => h,
|
||||
Err(e) => return super::error_exec(e.to_string()),
|
||||
};
|
||||
let start = args["start_line"].as_u64().map(|n| (n as usize).saturating_sub(1)).unwrap_or(0);
|
||||
let end_line = args["end_line"].as_u64().map(|n| n as usize);
|
||||
let limit = args["limit"].as_u64().map(|n| n.min(2000) as usize);
|
||||
return Box::new(SimpleExecution::new(Box::pin(async move {
|
||||
// A recognized medium is handed back for native inlining rather than
|
||||
// failing on non-UTF-8 bytes. We always emit the media (the message
|
||||
// builder gates on the resolved model's capability), so on a model
|
||||
// without the modality the note stands alone — never a decode error.
|
||||
if let Some(mime) = crate::session::handler::media::probe_media(&host).await {
|
||||
let size = tokio::fs::metadata(&host).await.map(|m| m.len()).unwrap_or(0);
|
||||
let host_str = host.to_string_lossy().into_owned();
|
||||
return Ok(ToolResult::Media {
|
||||
text: media_note(&path, mime, size),
|
||||
media: vec![MediaRef { host_path: host_str, mime: mime.to_string() }],
|
||||
});
|
||||
}
|
||||
let content = tokio::fs::read_to_string(&host).await
|
||||
.with_context(|| format!("Cannot read file: {path}"))?;
|
||||
Ok(ToolResult::Text(number_lines(&content, start, end_line, limit)))
|
||||
})));
|
||||
};
|
||||
let pool = match m.scope {
|
||||
MemScope::User => Arc::clone(&ctx.pool),
|
||||
|
||||
@@ -53,7 +53,7 @@ use anyhow::Result;
|
||||
use serde_json::Value;
|
||||
|
||||
pub use core_api::tool::{
|
||||
drive_execution, ExecutionOutcome, SimpleExecution, Tool, ToolCategory, ToolContext,
|
||||
drive_execution, ExecutionOutcome, MediaRef, SimpleExecution, Tool, ToolCategory, ToolContext,
|
||||
ToolDescriptionLength, ToolExecution, ToolResult, truncate_label,
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user