feat(chat): view context — tell the assistant what you're looking at
Nightly Build / build (push) Successful in 7m51s
Nightly Build / build (push) Successful in 7m51s
An eye next to the paperclip shares what the user has open with their next
message: the page, the folder being browsed, the file open in the viewer and
any highlighted passage (line numbers where a source view exists), plus which
entity a detail page is about. The bag is client-authored {label, value} pairs
in English — the backend only clamps (chars, never bytes), neutralizes the
harness tag and renders one <system-extra> block per message, deduped
consecutively so it appears exactly when the view changed. On by default,
per-device toggle, hover/tap to preview, a chip on every sent message;
docs/view-context.md for users, an updated harness.md clause for the model.
This commit is contained in:
@@ -18,13 +18,22 @@
|
||||
//!
|
||||
//! Both are re-checked here even though the paths came from trusted code: the
|
||||
//! container is writable by the agent, so any host-side read must re-verify.
|
||||
//!
|
||||
//! The same type also implements `agent_loop::projection::MessageExtras` — the
|
||||
//! **single** composer of a message's `<system-extra>` block (skipped attachment
|
||||
//! paths + the view context). One type, one `Arc`, two hooks: the block's first
|
||||
//! half is a media answer, so splitting them across two objects would mean
|
||||
//! either two blocks or a handle passed between them.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use agent_loop::projection::{MediaBlob, MediaSource};
|
||||
use agent_loop::projection::{MediaBlob, MediaSource, MessageExtras};
|
||||
use agent_loop::store::{StoredCall, StoredMessage};
|
||||
use core_api::message_meta::{Attachment, MessageMetadata, attachments_block};
|
||||
use core_api::message_meta::{
|
||||
Attachment, MessageMetadata, ViewContextItem, attachments_body, sanitize_view_context,
|
||||
system_extra, view_context_body,
|
||||
};
|
||||
use core_api::tool::MediaRef;
|
||||
use core_api::user_fs::{UPLOADS_SUBDIR, UserFs};
|
||||
use tracing::debug;
|
||||
@@ -136,21 +145,32 @@ impl SkaldMediaSource {
|
||||
Self { fs }
|
||||
}
|
||||
|
||||
/// The attachments a stored message carries, in wire order.
|
||||
fn attachments(msg: &StoredMessage) -> Vec<Attachment> {
|
||||
/// The message's metadata bag, or the empty one.
|
||||
fn meta(msg: &StoredMessage) -> MessageMetadata {
|
||||
msg.metadata
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<MessageMetadata>(v.clone()).ok())
|
||||
.map(|m| m.attachments)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The attachments a stored message carries, in wire order.
|
||||
fn attachments(msg: &StoredMessage) -> Vec<Attachment> {
|
||||
Self::meta(msg).attachments
|
||||
}
|
||||
|
||||
/// The view context a stored message carries, **canonicalized**: the clamp
|
||||
/// runs at the ingress, but a row written by an older build or another
|
||||
/// client has not been through it, and it is also what makes the dedupe
|
||||
/// compare like with like.
|
||||
fn view_context(msg: &StoredMessage) -> Vec<ViewContextItem> {
|
||||
sanitize_view_context(Self::meta(msg).view_context)
|
||||
}
|
||||
}
|
||||
|
||||
#[agent_loop::async_trait]
|
||||
impl MediaSource for SkaldMediaSource {
|
||||
async fn message_media(&self, msg: &StoredMessage) -> Vec<Arc<dyn MediaBlob>> {
|
||||
// Positions matter: `skipped_text` indexes this same list.
|
||||
// Positions matter: the `MessageExtras` impl below indexes this same list.
|
||||
attachment_blobs(&self.fs, &Self::attachments(msg))
|
||||
}
|
||||
|
||||
@@ -165,20 +185,58 @@ impl MediaSource for SkaldMediaSource {
|
||||
ref_blobs(&self.fs, &refs)
|
||||
}
|
||||
|
||||
fn skipped_text(&self, msg: &StoredMessage, skipped: &[usize]) -> Option<String> {
|
||||
if skipped.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
/// The one composer of a message's `<system-extra>` block.
|
||||
///
|
||||
/// Registered as the same `Arc` that serves [`MediaSource`], because the two
|
||||
/// halves need the same knowledge: which attachments were left out is a media
|
||||
/// answer, and it belongs in the same block as the view context. **One block per
|
||||
/// message** — attachments first, then the view — because two would read to the
|
||||
/// model as two unrelated harness interjections.
|
||||
#[agent_loop::async_trait]
|
||||
impl MessageExtras for SkaldMediaSource {
|
||||
async fn appended_text(
|
||||
&self,
|
||||
msg: &StoredMessage,
|
||||
prev: Option<&StoredMessage>,
|
||||
skipped: &[usize],
|
||||
) -> Option<String> {
|
||||
let mut bodies: Vec<String> = Vec::new();
|
||||
|
||||
// The media that did not make it: the agent can still read these with a
|
||||
// tool, so the paths go in as text.
|
||||
if !skipped.is_empty() {
|
||||
let attachments = Self::attachments(msg);
|
||||
let left: Vec<Attachment> = skipped
|
||||
.iter()
|
||||
.filter_map(|&i| attachments.get(i).cloned())
|
||||
.collect();
|
||||
let body = attachments_body(&left);
|
||||
if !body.is_empty() {
|
||||
bodies.push(body);
|
||||
}
|
||||
}
|
||||
let attachments = Self::attachments(msg);
|
||||
let left: Vec<Attachment> = skipped
|
||||
.iter()
|
||||
.filter_map(|&i| attachments.get(i).cloned())
|
||||
.collect();
|
||||
if left.is_empty() {
|
||||
return None;
|
||||
|
||||
// What the user was looking at — **unless the previous thing they said
|
||||
// was sent from the same view**. Consecutive dedupe: in the normal case
|
||||
// the page does not change between two messages, so this drops nearly
|
||||
// all of the noise and turns the block into a signal of *change*. Note
|
||||
// what it deliberately is not: it does not look at attachments (two
|
||||
// messages from one page with different files still list the files), it
|
||||
// re-emits on `prev == None` (after a compaction or a window cut the
|
||||
// model has lost the earlier block), and a message *without* a view
|
||||
// never suppresses anything — nothing here says "no longer shared", that
|
||||
// is the header's temporal clause's job.
|
||||
let view = Self::view_context(msg);
|
||||
if !view.is_empty() && !prev.is_some_and(|p| Self::view_context(p) == view) {
|
||||
let body = view_context_body(&view);
|
||||
if !body.is_empty() {
|
||||
bodies.push(body);
|
||||
}
|
||||
}
|
||||
// The textual path block: the agent can still read these with a tool.
|
||||
Some(attachments_block(&left))
|
||||
|
||||
(!bodies.is_empty()).then(|| system_extra(&bodies.join("\n\n")))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -343,4 +401,121 @@ mod tests {
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&tmp).await;
|
||||
}
|
||||
|
||||
// ── The `<system-extra>` composer ─────────────────────────────────────────
|
||||
|
||||
use agent_loop::ids::MessageId;
|
||||
use agent_loop::store::Role;
|
||||
use core_api::message_meta::ViewContextItem;
|
||||
|
||||
fn vc(label: &str, value: &str) -> ViewContextItem {
|
||||
ViewContextItem { label: label.into(), value: value.into() }
|
||||
}
|
||||
|
||||
/// A stored user message carrying `metadata`, and nothing else that matters.
|
||||
fn msg(meta: MessageMetadata) -> StoredMessage {
|
||||
StoredMessage {
|
||||
id: MessageId(1),
|
||||
role: Role::User,
|
||||
content: "hi".into(),
|
||||
reasoning: None,
|
||||
synthetic: false,
|
||||
failed: false,
|
||||
metadata: Some(serde_json::to_value(meta).unwrap()),
|
||||
usage: Default::default(),
|
||||
calls: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn source() -> SkaldMediaSource {
|
||||
SkaldMediaSource::new(Arc::new(fs_home(Path::new("/nonexistent/homes/u1"))))
|
||||
}
|
||||
|
||||
fn open_tag() -> String {
|
||||
format!("<{TAG}>", TAG = core_api::message_meta::SYSTEM_EXTRA_TAG)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn view_context_alone_produces_the_block() {
|
||||
let m = msg(MessageMetadata {
|
||||
view_context: vec![vc("Open page", "Files (#files)")],
|
||||
..Default::default()
|
||||
});
|
||||
// `skipped` empty: the message has no media at all.
|
||||
let out = source().appended_text(&m, None, &[]).await.unwrap();
|
||||
assert!(out.starts_with("\n\n"), "{out:?}");
|
||||
assert!(out.contains("Viewing at the time of this message:"));
|
||||
assert!(out.contains("* Open page: Files (#files)"));
|
||||
assert_eq!(out.matches(&open_tag()).count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attachments_and_view_share_one_block_attachments_first() {
|
||||
let m = msg(MessageMetadata {
|
||||
attachments: vec![att("uploads/1/a.png")],
|
||||
view_context: vec![vc("Open page", "Files (#files)")],
|
||||
..Default::default()
|
||||
});
|
||||
let out = source().appended_text(&m, None, &[0]).await.unwrap();
|
||||
assert_eq!(out.matches(&open_tag()).count(), 1, "exactly one block: {out}");
|
||||
let at = out.find("1 attached file:").unwrap();
|
||||
let view = out.find("Viewing at the time").unwrap();
|
||||
assert!(at < view, "attachments first: {out}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nothing_to_say_appends_nothing() {
|
||||
assert!(source().appended_text(&msg(MessageMetadata::default()), None, &[]).await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_dedupe_is_consecutive_and_structural() {
|
||||
let bag = vec![vc("Open page", "Files (#files)"), vc("Open folder", "shared/casa")];
|
||||
let same = msg(MessageMetadata { view_context: bag.clone(), ..Default::default() });
|
||||
let other = msg(MessageMetadata {
|
||||
view_context: vec![vc("Open page", "Projects (#projects)")],
|
||||
..Default::default()
|
||||
});
|
||||
let src = source();
|
||||
|
||||
// prev = None ⇒ emitted (a compaction or a window cut lands here).
|
||||
assert!(src.appended_text(&same, None, &[]).await.is_some());
|
||||
// Identical bag ⇒ suppressed.
|
||||
assert!(src.appended_text(&same, Some(&same), &[]).await.is_none());
|
||||
// Different bag ⇒ emitted.
|
||||
assert!(src.appended_text(&same, Some(&other), &[]).await.is_some());
|
||||
// A previous message with no view suppresses nothing.
|
||||
assert!(
|
||||
src.appended_text(&same, Some(&msg(MessageMetadata::default())), &[])
|
||||
.await
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_dedupe_ignores_attachments() {
|
||||
let bag = vec![vc("Open page", "Files (#files)")];
|
||||
let prev = msg(MessageMetadata { view_context: bag.clone(), ..Default::default() });
|
||||
let now = msg(MessageMetadata {
|
||||
attachments: vec![att("uploads/1/a.png")],
|
||||
view_context: bag,
|
||||
..Default::default()
|
||||
});
|
||||
let out = source().appended_text(&now, Some(&prev), &[0]).await.unwrap();
|
||||
assert!(out.contains("1 attached file:"), "{out}");
|
||||
assert!(!out.contains("Viewing at the time"), "view suppressed, files not: {out}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_message_with_only_skipped_media_is_byte_identical_to_before() {
|
||||
let m = msg(MessageMetadata {
|
||||
attachments: vec![att("uploads/1/a.png"), att("uploads/1/b.pdf")],
|
||||
..Default::default()
|
||||
});
|
||||
let out = source().appended_text(&m, None, &[0, 1]).await.unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
"\n\n<system-extra>\n2 attached files:\n* uploads/1/a.png\n* uploads/1/b.pdf\n</system-extra>"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,10 +65,16 @@ pub fn skald_projection(
|
||||
}
|
||||
}
|
||||
|
||||
/// The assembler every Skald turn runs on: the configuration above plus the two
|
||||
/// The assembler every Skald turn runs on: the configuration above plus the
|
||||
/// content hooks. `fs` is the caller's filesystem view — without it media is
|
||||
/// never inlined (nothing can be authorized), which is the right default for a
|
||||
/// context with no user workspace.
|
||||
///
|
||||
/// `SkaldMediaSource` is registered on **two** hooks from one `Arc`: it decides
|
||||
/// what may be inlined *and* composes the `<system-extra>` block. The block
|
||||
/// therefore rides on `fs` being present — true on every real path (both live
|
||||
/// call sites pass `Some`), and a context with no workspace has no view to
|
||||
/// describe either.
|
||||
pub fn skald_assembler(
|
||||
activation: Arc<dyn ActivationSource>,
|
||||
fs: Option<Arc<UserFs>>,
|
||||
@@ -85,7 +91,8 @@ pub fn skald_assembler(
|
||||
.with_activation(activation)
|
||||
.with_digest(Arc::new(SkaldDigest));
|
||||
if let Some(fs) = fs {
|
||||
assembler = assembler.with_media(Arc::new(SkaldMediaSource::new(fs)));
|
||||
let source = Arc::new(SkaldMediaSource::new(fs));
|
||||
assembler = assembler.with_media(source.clone()).with_extras(source);
|
||||
}
|
||||
assembler
|
||||
}
|
||||
|
||||
@@ -119,12 +119,18 @@ impl EventTranslator {
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value(v.clone()).ok());
|
||||
let attachments = meta.as_ref().map(|m| m.attachments.clone()).unwrap_or_default();
|
||||
let view_context = meta.as_ref().map(|m| m.view_context.clone()).unwrap_or_default();
|
||||
// A custom slash command persists its expanded template (for
|
||||
// LLM replay) but the bubble shows the typed command.
|
||||
let echo = meta
|
||||
.and_then(|m| m.command.map(|c| c.display))
|
||||
.unwrap_or(content);
|
||||
self.emit(ServerEvent::UserMessage { message_id: message_id.get(), content: echo, attachments }).await;
|
||||
self.emit(ServerEvent::UserMessage {
|
||||
message_id: message_id.get(),
|
||||
content: echo,
|
||||
attachments,
|
||||
view_context,
|
||||
}).await;
|
||||
}
|
||||
|
||||
LoopEvent::TokenDelta { kind, text } => {
|
||||
|
||||
Reference in New Issue
Block a user