diff --git a/agents/common/tools.md b/agents/common/tools.md index 9c6e6c4..4e83ef2 100644 --- a/agents/common/tools.md +++ b/agents/common/tools.md @@ -1,3 +1,7 @@ # Tools Scratchpad notes (`update_scratchpad`) are shared across all agents in the session and injected into every agent's context. Not persisted across sessions. Keep values concise. For a **private** task list that sub-agents should *not* see, use `write_todos` instead. + +## Understanding code before you read it + +When you need to understand source code you don't already know, reach for `get_ast_outline` **before** `read_file` — especially on a large file. It returns the file's structure and the line range of every definition at a fraction of the tokens. Then `read_file` only the ranges you actually need. Reading a whole unfamiliar file wastes context; outline first, read narrow. (`list_files` with `with_metadata=true` reports each file's size and line count, so you can spot which files are worth outlining.) diff --git a/crates/skald-core/src/db/memory_docs.rs b/crates/skald-core/src/db/memory_docs.rs index 662752f..ff2b616 100644 --- a/crates/skald-core/src/db/memory_docs.rs +++ b/crates/skald-core/src/db/memory_docs.rs @@ -34,6 +34,16 @@ pub struct MemoryHit { pub snippet: String, } +/// A directory listing row carrying cheap size metadata. `line_count` and +/// `byte_len` are computed in SQL (`LENGTH` / newline count) so the note body +/// never leaves the database. +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct MemoryEntryMeta { + pub path: String, + pub line_count: i64, + pub byte_len: i64, +} + const SELECT: &str = "SELECT id, path, content, created_at, updated_at FROM memory_docs"; /// Fetch one note by its exact path. @@ -84,6 +94,27 @@ pub async fn list(pool: &SqlitePool, prefix: &str) -> Result> { Ok(rows) } +/// Like [`list`], but each row also carries a line count and byte length, +/// computed in SQL so the body is never transferred. Line count matches the +/// on-disk convention: an empty note is 0 lines, otherwise newline-count + 1. +pub async fn list_with_metadata(pool: &SqlitePool, prefix: &str) -> Result> { + let pattern = format!("{}%", escape_like(prefix)); + let rows = sqlx::query_as::<_, MemoryEntryMeta>( + "SELECT path, + CASE WHEN content = '' THEN 0 + ELSE LENGTH(content) - LENGTH(REPLACE(content, char(10), '')) + 1 + END AS line_count, + LENGTH(CAST(content AS BLOB)) AS byte_len + FROM memory_docs + WHERE path LIKE ? ESCAPE '\\' + ORDER BY updated_at DESC", + ) + .bind(pattern) + .fetch_all(pool) + .await?; + Ok(rows) +} + /// Full-text search over note bodies and paths, best match first. `query` is /// FTS5 MATCH syntax; `snippet` is a short excerpt of the body with the matched /// terms wrapped in `[` … `]`. diff --git a/crates/skald-core/src/session/handler/message_builder.rs b/crates/skald-core/src/session/handler/message_builder.rs index 6e4c78c..025e3e0 100644 --- a/crates/skald-core/src/session/handler/message_builder.rs +++ b/crates/skald-core/src/session/handler/message_builder.rs @@ -618,9 +618,13 @@ impl MessageBuilder { } if !active.is_empty() { - out.push_str("\n**Active** — tools callable as `mcp____`:\n"); + out.push_str("\n**Active** — tools callable as `mcp____`:\n\n"); + out.push_str("| Server | Description |\n|--------|-------------|\n"); for name in &active { - out.push_str(&format!("- `{name}`\n")); + let desc = descriptions.get(*name) + .and_then(|d| d.as_deref()) + .unwrap_or("—"); + out.push_str(&format!("| `{name}` | {desc} |\n")); } } diff --git a/crates/skald-core/src/tools/ast_outline.rs b/crates/skald-core/src/tools/ast_outline.rs index 13d6e9d..e0800b0 100644 --- a/crates/skald-core/src/tools/ast_outline.rs +++ b/crates/skald-core/src/tools/ast_outline.rs @@ -15,12 +15,14 @@ impl Tool for AstOutline { fn category(&self) -> crate::tools::ToolCategory { crate::tools::ToolCategory::Filesystem } fn description(&self) -> &str { - "Return the structural outline of a source file: top-level definitions (functions, classes, \ - structs, methods, traits, interfaces, etc.) without their bodies. \ - Each entry is formatted as 'START-END | : ' where START and END are 1-based \ - line numbers of the full definition — same column format as read_file, so you can pass \ - START/END directly to read_file's start_line/end_line to read just that definition. \ - Much cheaper than reading the full file when you only need to understand the shape of the code. \ + "Start here when you need to understand a source file you don't already know — especially a large one. \ + Returns the file's structural outline: top-level definitions (functions, classes, structs, methods, \ + traits, interfaces, etc.) without their bodies, so you grasp the whole shape at a fraction of the \ + tokens of reading it. \ + Each entry is formatted as 'START-END | : ' where START and END are 1-based line numbers \ + of the full definition — same column format as read_file, so you pass START/END straight to \ + read_file's start_line/end_line to read just the definition you care about. \ + Typical flow: outline first, then read only the ranges you need — far cheaper than reading the whole file. \ Supported: .rs .py .js .mjs .ts .tsx .go .java .c .h .cpp .cc .hpp .swift .lua .rb .sh .ex .exs \ .kt .json .toml .yaml .yml .html .css .md .sql" } @@ -67,7 +69,7 @@ impl Tool for AstOutline { "rb" => outline_ts(path, ts_ruby(), "Ruby"), "sh" | "bash" => outline_ts(path, ts_bash(), "Bash"), "ex" | "exs" => outline_ts(path, ts_elixir(), "Elixir"), - "json" => outline_ts(path, ts_json(), "JSON"), + "json" => outline_json(path), "yaml" | "yml" => outline_ts(path, ts_yaml(), "YAML"), "html" => outline_ts(path, ts_html(), "HTML"), "css" => outline_ts(path, ts_css(), "CSS"), @@ -292,15 +294,6 @@ fn ts_elixir() -> LangConfig { } } -fn ts_json() -> LangConfig { - LangConfig { - language: tree_sitter_json::LANGUAGE.into(), - def_kinds: &["pair"], - name_field: "key", - container_kinds: &[], - } -} - fn ts_yaml() -> LangConfig { LangConfig { language: tree_sitter_yaml::LANGUAGE.into(), @@ -331,6 +324,147 @@ fn ts_css() -> LangConfig { } } +// ── JSON outline (dedicated tree-sitter walker: nested keys) ──────────────── +// +// The generic `collect_nodes` only descends through `container_kinds`, which for +// JSON tops out at the first level of the root object (and never enters arrays +// of objects). This walker recurses through the parse tree instead: it lists +// every key at every depth, shows scalar values inline, and expands nested +// objects/arrays. Line ranges keep the read_file contract (`START-END | …`). + +const JSON_VALUE_KINDS: &[&str] = + &["object", "array", "string", "number", "true", "false", "null"]; + +fn outline_json(path: &str) -> Result { + let source = read_to_string(path)?; + let mut parser = tree_sitter::Parser::new(); + let language: tree_sitter::Language = tree_sitter_json::LANGUAGE.into(); + parser.set_language(&language) + .map_err(|e| anyhow::anyhow!("tree-sitter language load error: {e}"))?; + let tree = parser.parse(source.as_bytes(), None) + .ok_or_else(|| anyhow::anyhow!("tree-sitter parse returned None for {path}"))?; + + let mut out = format!("--- JSON outline: {path} ---\n\n"); + // document → single top-level value (object or array). + if let Some(top) = json_first_value(tree.root_node()) { + json_walk(top, &source, 0, &mut out); + } + Ok(out) +} + +/// First JSON value child of `document` (skips comments/whitespace nodes). +fn json_first_value(document: tree_sitter::Node) -> Option { + for i in 0..document.child_count() { + let c = document.child(i as u32).unwrap(); + if JSON_VALUE_KINDS.contains(&c.kind()) { + return Some(c); + } + } + None +} + +/// Emit one line per entry of an object/array, recursing into nested containers. +/// Scalars are shown inline; scalar array elements are summarised by the array's +/// header only (not listed) to stay readable on large value arrays. +fn json_walk(node: tree_sitter::Node, source: &str, depth: usize, out: &mut String) { + const MAX_JSON_DEPTH: usize = 16; + if depth > MAX_JSON_DEPTH { + return; + } + match node.kind() { + "object" => { + for i in 0..node.child_count() { + let pair = node.child(i as u32).unwrap(); + if pair.kind() != "pair" { + continue; + } + let (Some(key), Some(val)) = ( + pair.child_by_field_name("key"), + pair.child_by_field_name("value"), + ) else { + continue; + }; + json_emit(&json_key_text(key, source), val, pair, source, depth, out); + } + } + "array" => { + let mut idx = 0usize; + for i in 0..node.child_count() { + let el = node.child(i as u32).unwrap(); + if !JSON_VALUE_KINDS.contains(&el.kind()) { + continue; + } + let this = idx; + idx += 1; + // Only expand container elements; scalars are covered by the count. + if el.kind() == "object" || el.kind() == "array" { + json_emit(&format!("[{this}]"), el, el, source, depth, out); + } + } + } + _ => {} + } +} + +/// Emit one entry line (`name: `) spanning `span`'s rows, +/// then recurse when the value is itself a container. +fn json_emit( + name: &str, + val: tree_sitter::Node, + span: tree_sitter::Node, + source: &str, + depth: usize, + out: &mut String, +) { + let start = span.start_position().row + 1; + let end = span.end_position().row + 1; + let indent = " ".repeat(depth); + let desc = json_value_desc(val, source); + out.push_str(&format!("{start:>4}-{end:>4} | {indent}{name}: {desc}\n")); + if val.kind() == "object" || val.kind() == "array" { + json_walk(val, source, depth + 1, out); + } +} + +/// Short descriptor of a value: `{N keys}`, `[N items]`, or the scalar literal. +fn json_value_desc(node: tree_sitter::Node, source: &str) -> String { + match node.kind() { + "object" => { + let n = json_count(node, &["pair"]); + format!("{{{n} {}}}", if n == 1 { "key" } else { "keys" }) + } + "array" => { + let n = json_count(node, JSON_VALUE_KINDS); + format!("[{n} {}]", if n == 1 { "item" } else { "items" }) + } + _ => { + let raw = source.get(node.byte_range()).unwrap_or(""); + let one = raw.split_whitespace().collect::>().join(" "); + truncate_label(&one, MAX_LABEL_SHORT) + } + } +} + +/// Number of direct children whose kind is in `kinds`. +fn json_count(node: tree_sitter::Node, kinds: &[&str]) -> usize { + let mut n = 0; + for i in 0..node.child_count() { + if kinds.contains(&node.child(i as u32).unwrap().kind()) { + n += 1; + } + } + n +} + +/// Object key text with the surrounding double-quotes stripped. +fn json_key_text(key: tree_sitter::Node, source: &str) -> String { + let raw = source.get(key.byte_range()).unwrap_or(""); + raw.strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .unwrap_or(raw) + .to_string() +} + // ── text-based fallbacks (crates incompatible with tree-sitter 0.26) ─────── fn outline_kotlin(path: &str) -> Result { diff --git a/crates/skald-core/src/tools/fs/list_files.rs b/crates/skald-core/src/tools/fs/list_files.rs index 0957b9a..075459a 100644 --- a/crates/skald-core/src/tools/fs/list_files.rs +++ b/crates/skald-core/src/tools/fs/list_files.rs @@ -36,6 +36,8 @@ impl Tool for ListFiles { Skips .git, target, node_modules, .cache. \ Returns a JSON array of paths relative to the requested directory. \ Use depth=1 for immediate contents only, depth=2-3 for moderate exploration. \ + Set with_metadata=true to instead return objects {path, line_count?, size} — handy for spotting a \ + large file worth outlining with get_ast_outline before you read it. \ Listing under user-memory/ (private) or shared-memory/ (shared) lists your memory notes instead of disk." } @@ -54,6 +56,10 @@ impl Tool for ListFiles { "dirs_only": { "type": "boolean", "description": "If true, return only directories and omit files (default false)." + }, + "with_metadata": { + "type": "boolean", + "description": "If true, return objects {path, line_count?, size} instead of bare path strings. size is human-readable; line_count is included only for text files (omitted for binaries and very large files). Use it to decide whether to get_ast_outline a large file before reading it." } } }) @@ -71,6 +77,7 @@ impl Tool for ListFiles { /// under the prefix is returned, keyed relative to the requested directory. fn run_with<'a>(&'a self, ctx: &ToolContext, args: Value) -> Box { let path = args["path"].as_str().unwrap_or("").to_string(); + let with_metadata = args["with_metadata"].as_bool().unwrap_or(false); let Some(m) = classify_memory(&path) else { return match super::rewrite_to_host(&ctx.fs, &path, args) { Ok(args) => self.run(args), @@ -87,6 +94,21 @@ impl Tool for ListFiles { // Treat `rel` as a directory prefix: match `rel/…` (or everything at // the root), then strip it so results are relative to what was asked. let prefix = if rel.is_empty() || rel.ends_with('/') { rel } else { format!("{rel}/") }; + + if with_metadata { + let mut entries: Vec = crate::db::memory_docs::list_with_metadata(&pool, &prefix) + .await? + .into_iter() + .map(|e| FileEntry { + path: e.path.strip_prefix(&prefix).unwrap_or(&e.path).to_string(), + line_count: Some(e.line_count.max(0) as usize), + size: Some(human_size(e.byte_len.max(0) as u64)), + }) + .collect(); + entries.sort_by(|a, b| a.path.cmp(&b.path)); + return Ok(ToolResult::Text(serde_json::to_string(&entries)?)); + } + let entries = crate::db::memory_docs::list(&pool, &prefix).await?; let mut paths: Vec = entries.into_iter() .map(|e| e.path.strip_prefix(&prefix).unwrap_or(&e.path).to_string()) @@ -100,12 +122,81 @@ impl Tool for ListFiles { let user_path = args["path"].as_str().unwrap_or("."); let max_depth = args["depth"].as_u64().unwrap_or(3) as usize; let dirs_only = args["dirs_only"].as_bool().unwrap_or(false); + let with_metadata = args["with_metadata"].as_bool().unwrap_or(false); let dir = resolve(user_path)?; let mut paths: Vec = Vec::new(); walk(&dir, &dir, 0, max_depth, dirs_only, &mut paths)?; paths.sort(); - Ok(serde_json::to_string(&paths)?) + + if !with_metadata { + return Ok(serde_json::to_string(&paths)?); + } + let entries: Vec = paths.into_iter() + .map(|rel| file_entry(&dir.join(&rel), rel)) + .collect(); + Ok(serde_json::to_string(&entries)?) + } +} + +/// A `with_metadata` listing row. Field order (declaration order) is the wire +/// order; `line_count` and `size` are omitted when unavailable. +#[derive(serde::Serialize)] +struct FileEntry { + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + line_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + size: Option, +} + +/// Largest file we'll read to count lines; bigger files report `size` only, so +/// a metadata listing never turns into a full read of the tree. +const LINE_COUNT_SIZE_CAP: u64 = 2 * 1024 * 1024; + +/// Build a metadata row for one on-disk path. `size` comes free from a stat; +/// `line_count` is read only for text files within the size cap. +fn file_entry(abs: &Path, rel: String) -> FileEntry { + let meta = std::fs::metadata(abs).ok(); + let len = meta.as_ref().map(|m| m.len()); + let is_file = meta.as_ref().map(|m| m.is_file()).unwrap_or(false); + let line_count = match len { + Some(l) if is_file && l <= LINE_COUNT_SIZE_CAP => count_lines_if_text(abs), + _ => None, + }; + FileEntry { path: rel, line_count, size: len.map(human_size) } +} + +/// Line count of a text file, or `None` if it reads as binary (contains a NUL). +fn count_lines_if_text(abs: &Path) -> Option { + let bytes = std::fs::read(abs).ok()?; + if bytes.contains(&0) { return None; } + Some(count_lines(&bytes)) +} + +/// Number of lines an editor would show: 0 for empty, else newline-count plus +/// one when the file does not end in a newline. +fn count_lines(bytes: &[u8]) -> usize { + if bytes.is_empty() { return 0; } + let nl = bytes.iter().filter(|&&b| b == b'\n').count(); + if bytes.last() == Some(&b'\n') { nl } else { nl + 1 } +} + +/// Human-readable byte size, `ls -h` style (base 1024): "512 B", "18 KB", "1.4 MB". +fn human_size(bytes: u64) -> String { + const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"]; + let mut size = bytes as f64; + let mut unit = 0; + while size >= 1024.0 && unit < UNITS.len() - 1 { + size /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} B") + } else { + let s = format!("{size:.1}"); + let s = s.strip_suffix(".0").unwrap_or(&s); + format!("{s} {}", UNITS[unit]) } } @@ -138,3 +229,43 @@ fn walk(root: &Path, dir: &Path, depth: usize, max_depth: usize, dirs_only: bool } Ok(()) } + +#[cfg(test)] +mod meta_smoke { + use super::*; + const SP: &str = "/private/tmp/claude-501/-Users-dguiducci-projects-skald-circle/1cb4c456-6a62-4c67-abf8-bb93ef73e30c/scratchpad/lf"; + + #[test] + fn human_size_fmt() { + assert_eq!(human_size(0), "0 B"); + assert_eq!(human_size(512), "512 B"); + assert_eq!(human_size(18 * 1024), "18 KB"); + assert_eq!(human_size(1024 * 1024 + 400 * 1024), "1.4 MB"); + } + + #[test] + fn line_counts() { + assert_eq!(count_lines(b""), 0); + assert_eq!(count_lines(b"a\nb\nc\n"), 3); + assert_eq!(count_lines(b"no newline"), 1); + } + + #[test] + fn entries() { + let t = std::path::Path::new(SP).join("three.txt"); + let e = file_entry(&t, "three.txt".into()); + println!("three.txt -> {}", serde_json::to_string(&e).unwrap()); + assert_eq!(e.line_count, Some(3)); + + let o = std::path::Path::new(SP).join("one.txt"); + let e = file_entry(&o, "one.txt".into()); + println!("one.txt -> {}", serde_json::to_string(&e).unwrap()); + assert_eq!(e.line_count, Some(1)); + + let b = std::path::Path::new(SP).join("blob.bin"); + let e = file_entry(&b, "blob.bin".into()); + println!("blob.bin -> {}", serde_json::to_string(&e).unwrap()); + assert_eq!(e.line_count, None); // binary: size only + assert!(e.size.is_some()); + } +} diff --git a/crates/skald-core/src/tools/fs/read_file.rs b/crates/skald-core/src/tools/fs/read_file.rs index 83b0d92..e0a8caa 100644 --- a/crates/skald-core/src/tools/fs/read_file.rs +++ b/crates/skald-core/src/tools/fs/read_file.rs @@ -78,6 +78,8 @@ impl Tool for ReadFile { Use instead of cat/head/tail in the terminal. \ Returns text prefixed as ' N | line'. When calling edit_file, copy the text after '| ' exactly. \ For large files use start_line/end_line to read in chunks — files over ~2000 lines should never be read whole. \ + For an unfamiliar source file, call get_ast_outline first to get each definition's line range, then read \ + just the range you need instead of the whole file. \ Use limit to cap output when end_line is unknown. \ Paths under user-memory/ (private) or shared-memory/ (shared) read a note from your memory instead of disk." }