feat(chat): view context — tell the assistant what you're looking at
Nightly Build / build (push) Successful in 7m51s

An eye next to the paperclip shares what the user has open with their next
message: the page, the folder being browsed, the file open in the viewer and
any highlighted passage (line numbers where a source view exists), plus which
entity a detail page is about. The bag is client-authored {label, value} pairs
in English — the backend only clamps (chars, never bytes), neutralizes the
harness tag and renders one <system-extra> block per message, deduped
consecutively so it appears exactly when the view changed. On by default,
per-device toggle, hover/tap to preview, a chip on every sent message;
docs/view-context.md for users, an updated harness.md clause for the model.
This commit is contained in:
Daniele
2026-08-23 20:53:30 +01:00
parent 488c702517
commit 505f2e95c1
42 changed files with 2096 additions and 122 deletions
@@ -18,13 +18,22 @@
//!
//! Both are re-checked here even though the paths came from trusted code: the
//! container is writable by the agent, so any host-side read must re-verify.
//!
//! The same type also implements `agent_loop::projection::MessageExtras` — the
//! **single** composer of a message's `<system-extra>` block (skipped attachment
//! paths + the view context). One type, one `Arc`, two hooks: the block's first
//! half is a media answer, so splitting them across two objects would mean
//! either two blocks or a handle passed between them.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use agent_loop::projection::{MediaBlob, MediaSource};
use agent_loop::projection::{MediaBlob, MediaSource, MessageExtras};
use agent_loop::store::{StoredCall, StoredMessage};
use core_api::message_meta::{Attachment, MessageMetadata, attachments_block};
use core_api::message_meta::{
Attachment, MessageMetadata, ViewContextItem, attachments_body, sanitize_view_context,
system_extra, view_context_body,
};
use core_api::tool::MediaRef;
use core_api::user_fs::{UPLOADS_SUBDIR, UserFs};
use tracing::debug;
@@ -136,21 +145,32 @@ impl SkaldMediaSource {
Self { fs }
}
/// The attachments a stored message carries, in wire order.
fn attachments(msg: &StoredMessage) -> Vec<Attachment> {
/// The message's metadata bag, or the empty one.
fn meta(msg: &StoredMessage) -> MessageMetadata {
msg.metadata
.as_ref()
.and_then(|v| serde_json::from_value::<MessageMetadata>(v.clone()).ok())
.map(|m| m.attachments)
.unwrap_or_default()
}
/// The attachments a stored message carries, in wire order.
fn attachments(msg: &StoredMessage) -> Vec<Attachment> {
Self::meta(msg).attachments
}
/// The view context a stored message carries, **canonicalized**: the clamp
/// runs at the ingress, but a row written by an older build or another
/// client has not been through it, and it is also what makes the dedupe
/// compare like with like.
fn view_context(msg: &StoredMessage) -> Vec<ViewContextItem> {
sanitize_view_context(Self::meta(msg).view_context)
}
}
#[agent_loop::async_trait]
impl MediaSource for SkaldMediaSource {
async fn message_media(&self, msg: &StoredMessage) -> Vec<Arc<dyn MediaBlob>> {
// Positions matter: `skipped_text` indexes this same list.
// Positions matter: the `MessageExtras` impl below indexes this same list.
attachment_blobs(&self.fs, &Self::attachments(msg))
}
@@ -165,20 +185,58 @@ impl MediaSource for SkaldMediaSource {
ref_blobs(&self.fs, &refs)
}
fn skipped_text(&self, msg: &StoredMessage, skipped: &[usize]) -> Option<String> {
if skipped.is_empty() {
return None;
}
/// The one composer of a message's `<system-extra>` block.
///
/// Registered as the same `Arc` that serves [`MediaSource`], because the two
/// halves need the same knowledge: which attachments were left out is a media
/// answer, and it belongs in the same block as the view context. **One block per
/// message** — attachments first, then the view — because two would read to the
/// model as two unrelated harness interjections.
#[agent_loop::async_trait]
impl MessageExtras for SkaldMediaSource {
async fn appended_text(
&self,
msg: &StoredMessage,
prev: Option<&StoredMessage>,
skipped: &[usize],
) -> Option<String> {
let mut bodies: Vec<String> = Vec::new();
// The media that did not make it: the agent can still read these with a
// tool, so the paths go in as text.
if !skipped.is_empty() {
let attachments = Self::attachments(msg);
let left: Vec<Attachment> = skipped
.iter()
.filter_map(|&i| attachments.get(i).cloned())
.collect();
let body = attachments_body(&left);
if !body.is_empty() {
bodies.push(body);
}
}
let attachments = Self::attachments(msg);
let left: Vec<Attachment> = skipped
.iter()
.filter_map(|&i| attachments.get(i).cloned())
.collect();
if left.is_empty() {
return None;
// What the user was looking at — **unless the previous thing they said
// was sent from the same view**. Consecutive dedupe: in the normal case
// the page does not change between two messages, so this drops nearly
// all of the noise and turns the block into a signal of *change*. Note
// what it deliberately is not: it does not look at attachments (two
// messages from one page with different files still list the files), it
// re-emits on `prev == None` (after a compaction or a window cut the
// model has lost the earlier block), and a message *without* a view
// never suppresses anything — nothing here says "no longer shared", that
// is the header's temporal clause's job.
let view = Self::view_context(msg);
if !view.is_empty() && !prev.is_some_and(|p| Self::view_context(p) == view) {
let body = view_context_body(&view);
if !body.is_empty() {
bodies.push(body);
}
}
// The textual path block: the agent can still read these with a tool.
Some(attachments_block(&left))
(!bodies.is_empty()).then(|| system_extra(&bodies.join("\n\n")))
}
}
@@ -343,4 +401,121 @@ mod tests {
let _ = tokio::fs::remove_dir_all(&tmp).await;
}
// ── The `<system-extra>` composer ─────────────────────────────────────────
use agent_loop::ids::MessageId;
use agent_loop::store::Role;
use core_api::message_meta::ViewContextItem;
fn vc(label: &str, value: &str) -> ViewContextItem {
ViewContextItem { label: label.into(), value: value.into() }
}
/// A stored user message carrying `metadata`, and nothing else that matters.
fn msg(meta: MessageMetadata) -> StoredMessage {
StoredMessage {
id: MessageId(1),
role: Role::User,
content: "hi".into(),
reasoning: None,
synthetic: false,
failed: false,
metadata: Some(serde_json::to_value(meta).unwrap()),
usage: Default::default(),
calls: vec![],
}
}
fn source() -> SkaldMediaSource {
SkaldMediaSource::new(Arc::new(fs_home(Path::new("/nonexistent/homes/u1"))))
}
fn open_tag() -> String {
format!("<{TAG}>", TAG = core_api::message_meta::SYSTEM_EXTRA_TAG)
}
#[tokio::test]
async fn view_context_alone_produces_the_block() {
let m = msg(MessageMetadata {
view_context: vec![vc("Open page", "Files (#files)")],
..Default::default()
});
// `skipped` empty: the message has no media at all.
let out = source().appended_text(&m, None, &[]).await.unwrap();
assert!(out.starts_with("\n\n"), "{out:?}");
assert!(out.contains("Viewing at the time of this message:"));
assert!(out.contains("* Open page: Files (#files)"));
assert_eq!(out.matches(&open_tag()).count(), 1);
}
#[tokio::test]
async fn attachments_and_view_share_one_block_attachments_first() {
let m = msg(MessageMetadata {
attachments: vec![att("uploads/1/a.png")],
view_context: vec![vc("Open page", "Files (#files)")],
..Default::default()
});
let out = source().appended_text(&m, None, &[0]).await.unwrap();
assert_eq!(out.matches(&open_tag()).count(), 1, "exactly one block: {out}");
let at = out.find("1 attached file:").unwrap();
let view = out.find("Viewing at the time").unwrap();
assert!(at < view, "attachments first: {out}");
}
#[tokio::test]
async fn nothing_to_say_appends_nothing() {
assert!(source().appended_text(&msg(MessageMetadata::default()), None, &[]).await.is_none());
}
#[tokio::test]
async fn the_dedupe_is_consecutive_and_structural() {
let bag = vec![vc("Open page", "Files (#files)"), vc("Open folder", "shared/casa")];
let same = msg(MessageMetadata { view_context: bag.clone(), ..Default::default() });
let other = msg(MessageMetadata {
view_context: vec![vc("Open page", "Projects (#projects)")],
..Default::default()
});
let src = source();
// prev = None ⇒ emitted (a compaction or a window cut lands here).
assert!(src.appended_text(&same, None, &[]).await.is_some());
// Identical bag ⇒ suppressed.
assert!(src.appended_text(&same, Some(&same), &[]).await.is_none());
// Different bag ⇒ emitted.
assert!(src.appended_text(&same, Some(&other), &[]).await.is_some());
// A previous message with no view suppresses nothing.
assert!(
src.appended_text(&same, Some(&msg(MessageMetadata::default())), &[])
.await
.is_some()
);
}
#[tokio::test]
async fn the_dedupe_ignores_attachments() {
let bag = vec![vc("Open page", "Files (#files)")];
let prev = msg(MessageMetadata { view_context: bag.clone(), ..Default::default() });
let now = msg(MessageMetadata {
attachments: vec![att("uploads/1/a.png")],
view_context: bag,
..Default::default()
});
let out = source().appended_text(&now, Some(&prev), &[0]).await.unwrap();
assert!(out.contains("1 attached file:"), "{out}");
assert!(!out.contains("Viewing at the time"), "view suppressed, files not: {out}");
}
#[tokio::test]
async fn a_message_with_only_skipped_media_is_byte_identical_to_before() {
let m = msg(MessageMetadata {
attachments: vec![att("uploads/1/a.png"), att("uploads/1/b.pdf")],
..Default::default()
});
let out = source().appended_text(&m, None, &[0, 1]).await.unwrap();
assert_eq!(
out,
"\n\n<system-extra>\n2 attached files:\n* uploads/1/a.png\n* uploads/1/b.pdf\n</system-extra>"
);
}
}
@@ -65,10 +65,16 @@ pub fn skald_projection(
}
}
/// The assembler every Skald turn runs on: the configuration above plus the two
/// The assembler every Skald turn runs on: the configuration above plus the
/// content hooks. `fs` is the caller's filesystem view — without it media is
/// never inlined (nothing can be authorized), which is the right default for a
/// context with no user workspace.
///
/// `SkaldMediaSource` is registered on **two** hooks from one `Arc`: it decides
/// what may be inlined *and* composes the `<system-extra>` block. The block
/// therefore rides on `fs` being present — true on every real path (both live
/// call sites pass `Some`), and a context with no workspace has no view to
/// describe either.
pub fn skald_assembler(
activation: Arc<dyn ActivationSource>,
fs: Option<Arc<UserFs>>,
@@ -85,7 +91,8 @@ pub fn skald_assembler(
.with_activation(activation)
.with_digest(Arc::new(SkaldDigest));
if let Some(fs) = fs {
assembler = assembler.with_media(Arc::new(SkaldMediaSource::new(fs)));
let source = Arc::new(SkaldMediaSource::new(fs));
assembler = assembler.with_media(source.clone()).with_extras(source);
}
assembler
}
@@ -119,12 +119,18 @@ impl EventTranslator {
.as_ref()
.and_then(|v| serde_json::from_value(v.clone()).ok());
let attachments = meta.as_ref().map(|m| m.attachments.clone()).unwrap_or_default();
let view_context = meta.as_ref().map(|m| m.view_context.clone()).unwrap_or_default();
// A custom slash command persists its expanded template (for
// LLM replay) but the bubble shows the typed command.
let echo = meta
.and_then(|m| m.command.map(|c| c.display))
.unwrap_or(content);
self.emit(ServerEvent::UserMessage { message_id: message_id.get(), content: echo, attachments }).await;
self.emit(ServerEvent::UserMessage {
message_id: message_id.get(),
content: echo,
attachments,
view_context,
}).await;
}
LoopEvent::TokenDelta { kind, text } => {