//! Structured, reusable metadata attached to a `chat_history` row. //! //! Persisted as a single JSON column (`chat_history.metadata`) and intentionally //! generic: today it carries user file **attachments** and the **view context** //! (what the user was looking at), but new keys can be added later without a //! schema change. Two independent readers derive different views from the same //! source: //! - the **LLM context** builder appends [`attachments_body`] / //! [`view_context_body`] to the user turn, inside one `` block, //! - the **history UI** renders the structured metadata as chips. //! //! The raw `` text block is therefore never persisted — it is //! generated on the fly from this metadata. The tag name lives in //! [`SYSTEM_EXTRA_TAG`] so emission sites and the agent-facing instruction that //! documents it can never drift apart. //! //! The `*_body` functions return **unwrapped** text: a message gets exactly one //! `` block, so framing belongs to whoever composes it (in this //! workspace, `SkaldMediaSource`'s `MessageExtras` impl) and never to the pieces. use std::borrow::Cow; use serde::{Deserialize, Serialize}; /// One file attached by the user to a message. `path` is a home-relative agent /// path (e.g. `uploads/123/file.pdf`) — the caller's container home is its root, /// so the fs-tools, `execute_cmd`, the file viewer (`/api/file`) and the media /// inliner all resolve it to the same physical file. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Attachment { pub path: String, pub name: String, /// Best-effort MIME type (e.g. `application/pdf`); `None` if unknown. #[serde(default, skip_serializing_if = "Option::is_none")] pub mimetype: Option, /// Size in bytes, when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub filesize: Option, } /// One `{label, value}` pair describing a slice of what the user had on screen /// when the message was sent — the open page, the open folder, the selected text. /// /// **Both halves are opaque free text written by the client, in English.** The /// backend never matches on a label, never parses a value, and knows no key /// names: a new page is a row in the frontend's table and zero lines of Rust. /// Line numbers, entity names and the like are composed by the client *into the /// label* (`"Selected text (report.md, lines 12-17)"`) for exactly that reason. /// /// The list is ordered by the client and rendered in that order — a map would /// make rendering order an accident of key naming, and order is part of the /// provider's prefix-cache key. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ViewContextItem { pub label: String, pub value: String, } /// Generic metadata bag for a chat message. Extra keys may be added over time; /// `#[serde(default)]` keeps deserialization tolerant of older/newer shapes. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct MessageMetadata { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub attachments: Vec, /// Present when this user turn was produced by a custom slash command. #[serde(default, skip_serializing_if = "Option::is_none")] pub command: Option, /// What the user was looking at, as sent by the client and already put /// through [`sanitize_view_context`] at the ingress. Absent (empty) for every /// source that has no view — Telegram, cron, background agents. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub view_context: Vec, } impl MessageMetadata { /// True when there is nothing worth persisting. Every field must be listed /// here: a message carrying *only* view context would otherwise be stored /// with `metadata = NULL`. pub fn is_empty(&self) -> bool { self.attachments.is_empty() && self.command.is_none() && self.view_context.is_empty() } } /// Identifies a user message produced by a custom slash command. The history row's /// `content` holds the **expanded template** (replayed to the LLM verbatim); the UI /// renders `display` — the original `/command …` the user typed — instead. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct CommandRef { /// Canonical command name, without the leading `/` (e.g. `review`). pub name: String, /// The original text the user typed (e.g. `/review revisiona Cat.java`). pub display: String, } /// The canonical name of the tag that wraps harness-injected data (attachments, /// locations, transcripts, hook output…) inside user messages and tool results. /// /// Single source of truth: every emission site builds via [`system_extra`], and /// the agent-facing instruction that documents the tag interpolates this same /// constant (via the `__HARNESS_TAG__` substitution). Renaming the tag is a /// one-line change here. pub const SYSTEM_EXTRA_TAG: &str = "system-extra"; /// Wraps a harness-generated body in the canonical `` block, with /// a leading blank-line pair so it can be concatenated onto the tail of a user /// message or a tool result. Returns the full block (open tag, body, close tag). /// /// Callers must not add their own leading newlines — this helper owns the /// framing. An empty `body` still emits the (empty) block; callers that want a /// no-op on empty input check themselves — the `*_body` builders return `""` /// precisely so a composer can test before wrapping. pub fn system_extra(body: &str) -> String { format!("\n\n<{TAG}>\n{body}\n", TAG = SYSTEM_EXTRA_TAG) } /// Escapes the harness tag so a value can never break out of the block that /// carries it. Replaces `<` with `<` **only** in the two sequences /// `` and `` (case-insensitive), leaving every other /// `<` alone — the body is data the model reads, not markup we own. /// /// This is not a hypothetical: a selected paragraph, or a file written by another /// member in a shared folder, can contain the closing tag verbatim, and would /// then continue as if it were the user speaking. Applied to labels, values /// **and attachment paths** (a file may legitimately be named ``). pub fn neutralize_harness_tag(s: &str) -> Cow<'_, str> { let open = format!("<{TAG}>", TAG = SYSTEM_EXTRA_TAG); let close = format!("", TAG = SYSTEM_EXTRA_TAG); // ASCII-only lowercasing: byte-length preserving, so indices into `hay` are // valid indices into `s` (a Unicode `to_lowercase` is not). let hay = s.to_ascii_lowercase(); if !hay.contains(&open) && !hay.contains(&close) { return Cow::Borrowed(s); } let mut out = String::with_capacity(s.len() + 8); let mut i = 0usize; while i < s.len() { // `` cannot match at a ` Some((a, open.len())), (Some(_), Some(b)) => Some((b, close.len())), (Some(a), None) => Some((a, open.len())), (None, Some(b)) => Some((b, close.len())), (None, None) => None, }; match next { Some((rel, len)) => { let at = i + rel; out.push_str(&s[i..at]); out.push_str("<"); // Keep the rest of the tag verbatim, original casing included. out.push_str(&s[at + 1..at + len]); i = at + len; } None => { out.push_str(&s[i..]); break; } } } Cow::Owned(out) } // ── View-context caps ───────────────────────────────────────────────────────── // // A text selection is unbounded by nature: a Cmd+A on a 2 MB file would ride in // *every* future projection of that message, forever, at cost. So the bag is // clamped — truncated, never rejected, with an explicit marker so the model // knows there is more and can read the file with a tool. /// Maximum number of `{label, value}` pairs kept on one message. pub const VIEW_CONTEXT_MAX_ITEMS: usize = 12; /// Maximum length of one label, in `char`s. pub const VIEW_CONTEXT_MAX_LABEL: usize = 120; /// Maximum length of one value, in `char`s. pub const VIEW_CONTEXT_MAX_VALUE: usize = 4_096; /// Maximum sum of every label + value on one message, in `char`s. pub const VIEW_CONTEXT_MAX_TOTAL: usize = 16_384; /// Truncates to `max` **`char`s including the marker**, so the result is always /// within budget and a second pass leaves it alone (idempotence). fn clamp_chars(s: &str, max: usize) -> Cow<'_, str> { let total = s.chars().count(); if total <= max { return Cow::Borrowed(s); } let marker = |kept: usize| format!("… [truncated: {kept} of {total} characters]"); // Two passes: the marker's own length depends on the number it prints, and // the digit count can shrink once. Either way the result stays ≤ max. let mut kept = max.saturating_sub(marker(max).chars().count()); kept = max.saturating_sub(marker(kept).chars().count()); let head: String = s.chars().take(kept).collect(); Cow::Owned(format!("{head}{}", marker(kept))) } /// Canonicalises an inbound view-context bag: neutralize the tag, clamp each /// label, clamp each value, clamp the item count, clamp the running total. /// /// Applied **at the ingress** (so the megabyte is never persisted) and again at /// render time (old rows, other clients — defence in depth), which is why it is /// idempotent: sanitizing an already-sanitized bag returns it unchanged. pub fn sanitize_view_context(items: Vec) -> Vec { // Below this many chars of budget an item would be nothing but its own // truncation marker, so it is dropped instead. const MIN_VALUE_BUDGET: usize = 64; let mut out: Vec = Vec::with_capacity(items.len().min(VIEW_CONTEXT_MAX_ITEMS)); let mut used = 0usize; for item in items.into_iter().take(VIEW_CONTEXT_MAX_ITEMS) { let label = clamp_chars(&neutralize_harness_tag(&item.label), VIEW_CONTEXT_MAX_LABEL).into_owned(); let value = clamp_chars(&neutralize_harness_tag(&item.value), VIEW_CONTEXT_MAX_VALUE).into_owned(); let label_len = label.chars().count(); let value_len = value.chars().count(); if used + label_len + value_len <= VIEW_CONTEXT_MAX_TOTAL { used += label_len + value_len; out.push(ViewContextItem { label, value }); continue; } // The overflowing item: keep as much of its value as the budget allows, // then stop — everything after it would be arbitrary anyway. let budget = VIEW_CONTEXT_MAX_TOTAL.saturating_sub(used + label_len); if budget >= MIN_VALUE_BUDGET { let value = clamp_chars(&value, budget).into_owned(); out.push(ViewContextItem { label, value }); } break; } out } /// The attachments body — the lines listing attached paths, **without** the /// `` wrapper: wrapping belongs to whoever composes the block, so /// attachments and view context can share one. /// /// Returns an empty string when there are none, so callers can unconditionally /// concatenate. Shared by the web/mobile path and the Telegram plugin so every /// surface emits an identical format. pub fn attachments_body(attachments: &[Attachment]) -> String { if attachments.is_empty() { return String::new(); } let noun = if attachments.len() == 1 { "file" } else { "files" }; let mut body = format!("{} attached {}:", attachments.len(), noun); for a in attachments { body.push_str(&format!("\n* {}", neutralize_harness_tag(&a.path))); } body } /// Constant header introducing the view-context lines. /// /// **Owned by the backend, not by the client**: it is the temporal clause that /// stops the model from reading an old block as the current state, and no client /// may drop it. const VIEW_CONTEXT_HEADER: &str = "Viewing at the time of this message:"; /// The view-context body — the header plus one line per pair, **without** the /// `` wrapper (same reason as [`attachments_body`]). /// /// Empty in, empty out: an empty bag renders the empty string, never an orphan /// header. A single-line value renders inline (`* {label}: {value}`); a /// multi-line one goes into a fenced block at column 0, with a fence longer than /// any backtick run it contains. pub fn view_context_body(items: &[ViewContextItem]) -> String { if items.is_empty() { return String::new(); } let items = sanitize_view_context(items.to_vec()); if items.is_empty() { return String::new(); } let mut body = String::from(VIEW_CONTEXT_HEADER); for it in &items { if it.value.contains('\n') { let fence = "`".repeat(longest_backtick_run(&it.value).max(2) + 1); body.push_str(&format!("\n* {}:\n{fence}\n{}\n{fence}", it.label, it.value)); } else { body.push_str(&format!("\n* {}: {}", it.label, it.value)); } } body } /// Length of the longest run of consecutive backticks in `s` (0 if none). fn longest_backtick_run(s: &str) -> usize { let mut best = 0usize; let mut cur = 0usize; for c in s.chars() { if c == '`' { cur += 1; best = best.max(cur); } else { cur = 0; } } best } #[cfg(test)] mod tests { use super::*; #[test] fn system_extra_wraps_body_in_tag() { let out = system_extra("hello"); let open = format!("<{TAG}>", TAG = SYSTEM_EXTRA_TAG); let close = format!("", TAG = SYSTEM_EXTRA_TAG); assert!(out.starts_with("\n\n"), "leading blank-line pair: {:?}", out); assert!(out.contains(&open), "open tag missing: {:?}", out); assert!(out.contains(&close), "close tag missing: {:?}", out); assert_eq!(out, "\n\n\nhello\n"); } #[test] fn system_extra_tag_name_follows_constant() { // If this breaks, emission and the documented name have diverged: rename // via SYSTEM_EXTRA_TAG only, never by editing this string. assert_eq!(SYSTEM_EXTRA_TAG, "system-extra"); let out = system_extra("x"); let tag = SYSTEM_EXTRA_TAG; assert!(out.contains(&format!("<{tag}>")) && out.contains(&format!(""))); } #[test] fn attachments_body_empty_is_empty() { assert_eq!(attachments_body(&[]), ""); } #[test] fn attachments_body_lists_paths_and_pluralises() { let a = Attachment { path: "uploads/1/a.png".into(), name: "a.png".into(), mimetype: None, filesize: None, }; let b = Attachment { path: "uploads/1/b.pdf".into(), name: "b.pdf".into(), mimetype: None, filesize: None, }; assert_eq!( attachments_body(std::slice::from_ref(&a)), "1 attached file:\n* uploads/1/a.png" ); assert_eq!( attachments_body(&[a, b]), "2 attached files:\n* uploads/1/a.png\n* uploads/1/b.pdf" ); } // ── View context ────────────────────────────────────────────────────────── fn vc(label: &str, value: &str) -> ViewContextItem { ViewContextItem { label: label.into(), value: value.into() } } fn close_tag() -> String { format!("", TAG = SYSTEM_EXTRA_TAG) } #[test] fn view_context_body_empty_is_empty() { assert_eq!(view_context_body(&[]), ""); // A bag that sanitizes down to nothing is empty too — never an orphan header. assert!(!view_context_body(&[vc("Open page", "Files")]).is_empty()); } #[test] fn view_context_body_renders_header_and_single_line_pairs() { let out = view_context_body(&[ vc("Open page", "File viewer (#file_viewer)"), vc("Open file", "shared/casa/report.md"), ]); assert_eq!( out, "Viewing at the time of this message:\n\ * Open page: File viewer (#file_viewer)\n\ * Open file: shared/casa/report.md" ); // No wrapper: composing the block is the caller's job. assert!(!out.contains(&format!("<{TAG}>", TAG = SYSTEM_EXTRA_TAG))); } #[test] fn view_context_body_fences_multiline_values() { let out = view_context_body(&[vc("Selected text (lines 12-17)", "one\ntwo")]); assert!(out.contains("* Selected text (lines 12-17):\n```\none\ntwo\n```"), "{out}"); } #[test] fn view_context_body_fence_outgrows_contained_backticks() { // Four backticks inside ⇒ a five-backtick fence, at column 0. let out = view_context_body(&[vc("Selected text", "a\n````\nb")]); assert!(out.contains("\n`````\na\n````\nb\n`````"), "{out}"); assert_eq!(longest_backtick_run("a ``` b `` c"), 3); assert_eq!(longest_backtick_run("none"), 0); } #[test] fn neutralize_only_touches_the_two_tag_sequences() { assert!(matches!(neutralize_harness_tag("a < b
c"), Cow::Borrowed(_))); let s = format!("before {} after <{TAG}>", close_tag(), TAG = SYSTEM_EXTRA_TAG); let out = neutralize_harness_tag(&s); assert_eq!(out, "before </system-extra> after <system-extra>"); // Case-insensitive, casing of the rest preserved. assert_eq!(neutralize_harness_tag(""), "</SYSTEM-EXTRA>"); // Idempotent. assert_eq!(neutralize_harness_tag(&out), out); } #[test] fn sanitized_rendering_never_carries_a_live_closing_tag() { let close = close_tag(); let items = sanitize_view_context(vec![ vc(&format!("Selected text {close}"), &format!("evil {close} text")), ]); let body = view_context_body(&items); assert!(!body.contains(&close), "{body}"); assert!(body.contains("</system-extra>")); // …and the same for an attachment path: a file may be named like the tag. let a = Attachment { path: format!("uploads/1/{close}.txt"), name: "x.txt".into(), mimetype: None, filesize: None, }; let out = attachments_body(&[a]); assert!(!out.contains(&close), "{out}"); } #[test] fn clamp_truncates_per_item_on_char_boundaries_with_a_marker() { // Accents and emoji: cutting by bytes would split a code point. let value: String = "é🙂".repeat(4_000); let items = sanitize_view_context(vec![vc("Selected text", &value)]); let got = &items[0].value; assert!(got.chars().count() <= VIEW_CONTEXT_MAX_VALUE); // The marker reports the real length so the model knows there is more. assert!(got.contains(&format!("of {} characters]", value.chars().count())), "{got}"); assert!(got.starts_with("é🙂")); let label: String = "L".repeat(500); let items = sanitize_view_context(vec![vc(&label, "v")]); assert!(items[0].label.chars().count() <= VIEW_CONTEXT_MAX_LABEL); assert!(items[0].label.contains("truncated")); } #[test] fn clamp_caps_the_item_count() { let many: Vec<_> = (0..40).map(|i| vc(&format!("L{i}"), "v")).collect(); let out = sanitize_view_context(many); assert_eq!(out.len(), VIEW_CONTEXT_MAX_ITEMS); // Order preserved: the first N, not an arbitrary N. assert_eq!(out[0].label, "L0"); assert_eq!(out[VIEW_CONTEXT_MAX_ITEMS - 1].label, format!("L{}", VIEW_CONTEXT_MAX_ITEMS - 1)); } #[test] fn clamp_caps_the_running_total() { let big = "x".repeat(VIEW_CONTEXT_MAX_VALUE); let items: Vec<_> = (0..8).map(|i| vc(&format!("L{i}"), &big)).collect(); let out = sanitize_view_context(items); let total: usize = out.iter().map(|i| i.label.chars().count() + i.value.chars().count()).sum(); assert!(total <= VIEW_CONTEXT_MAX_TOTAL, "total {total}"); // Four 4 KiB values fit in 16 KiB; the fifth is what overflows. assert!(out.len() < 8); } #[test] fn sanitize_is_idempotent() { let value: String = "é🙂".repeat(4_000); let close = close_tag(); let mut items: Vec<_> = (0..30) .map(|i| vc(&format!("{close} L{i}"), &value)) .collect(); items.push(vc("short", "v")); let once = sanitize_view_context(items); let twice = sanitize_view_context(once.clone()); assert_eq!(once, twice); // Rendering re-applies the clamp: same output both ways (defence in depth). assert_eq!(view_context_body(&once), view_context_body(&twice)); } #[test] fn metadata_with_only_view_context_is_not_empty() { let meta = MessageMetadata { view_context: vec![vc("Open page", "Files")], ..Default::default() }; assert!(!meta.is_empty()); assert!(MessageMetadata::default().is_empty()); } #[test] fn metadata_round_trips_and_tolerates_older_json() { let meta = MessageMetadata { view_context: vec![vc("Open file", "shared/casa/report.md")], ..Default::default() }; let json = serde_json::to_string(&meta).unwrap(); assert_eq!(json, r#"{"view_context":[{"label":"Open file","value":"shared/casa/report.md"}]}"#); assert_eq!(serde_json::from_str::(&json).unwrap(), meta); // A row written before the field existed. let old = r#"{"attachments":[{"path":"uploads/1/a.png","name":"a.png"}]}"#; let back: MessageMetadata = serde_json::from_str(old).unwrap(); assert!(back.view_context.is_empty()); assert_eq!(back.attachments.len(), 1); } }