feat(chat): view context — tell the assistant what you're looking at
Nightly Build / build (push) Successful in 7m51s

An eye next to the paperclip shares what the user has open with their next
message: the page, the folder being browsed, the file open in the viewer and
any highlighted passage (line numbers where a source view exists), plus which
entity a detail page is about. The bag is client-authored {label, value} pairs
in English — the backend only clamps (chars, never bytes), neutralizes the
harness tag and renders one <system-extra> block per message, deduped
consecutively so it appears exactly when the view changed. On by default,
per-device toggle, hover/tap to preview, a chip on every sent message;
docs/view-context.md for users, an updated harness.md clause for the model.
This commit is contained in:
Daniele
2026-08-23 20:53:30 +01:00
parent 488c702517
commit 505f2e95c1
42 changed files with 2096 additions and 122 deletions
+119 -3
View File
@@ -10,7 +10,8 @@ use agent_loop::ids::{ConversationId, FrameId, MessageId};
use agent_loop::model::ModelInfo;
use agent_loop::prelude::async_trait;
use agent_loop::projection::{
MediaBlob, MediaSource, Projection, ReasoningEcho, ResultLimit, ToolResultDigest,
MediaBlob, MediaSource, MessageExtras, Projection, ReasoningEcho, ResultLimit,
ToolResultDigest,
};
use agent_loop::store::{
CallOutcome, FrameSpec, HistoryStore, NewCall, NewMessage, NewSummary, StoredCall,
@@ -432,8 +433,34 @@ impl MediaSource for Media {
async fn call_media(&self, _calls: &[StoredCall]) -> Vec<Arc<dyn MediaBlob>> {
vec![Arc::new(Png("tool.png"))]
}
fn skipped_text(&self, _msg: &StoredMessage, skipped: &[usize]) -> Option<String> {
(!skipped.is_empty()).then(|| format!("\n[files: {}]", skipped.len()))
}
/// The appended-text hook, in its own object: a note for the media left out, and
/// — whatever the media — the message's `extra` metadata key, so the tests can
/// tell "there was nothing to inline" from "there was nothing to say".
struct Extras;
#[async_trait]
impl MessageExtras for Extras {
async fn appended_text(
&self,
msg: &StoredMessage,
prev: Option<&StoredMessage>,
skipped: &[usize],
) -> Option<String> {
let mut out = String::new();
if !skipped.is_empty() {
out.push_str(&format!("\n[files: {}]", skipped.len()));
}
let extra = |m: &StoredMessage| {
m.metadata.as_ref().and_then(|v| v["extra"].as_str().map(str::to_string))
};
if let Some(e) = extra(msg)
&& prev.and_then(extra) != Some(e.clone())
{
out.push_str(&format!("\n[extra: {e}]"));
}
(!out.is_empty()).then_some(out)
}
}
@@ -446,6 +473,7 @@ async fn media_is_inlined_for_the_current_turn_and_textual_before_it() {
let msgs = LinearAssembler::new()
.with_media(Arc::new(Media))
.with_extras(Arc::new(Extras))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
capabilities: vec!["vision".into()],
..ModelInfo::default()
@@ -473,6 +501,7 @@ async fn a_model_without_vision_never_receives_bytes() {
let msgs = LinearAssembler::new()
.with_media(Arc::new(Media))
.with_extras(Arc::new(Extras))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
.await
.unwrap();
@@ -493,6 +522,7 @@ async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
let msgs = LinearAssembler::new()
.with_media(Arc::new(Media))
.with_extras(Arc::new(Extras))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
capabilities: vec!["vision".into()],
..ModelInfo::default()
@@ -505,3 +535,89 @@ async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
assert_eq!(last["content"][0]["type"], "image_url");
assert_eq!(msgs[msgs.len() - 2]["role"], "tool", "it follows the result group");
}
// ── Appended extras ──────────────────────────────────────────────────────────
/// The regression this hook exists for: as a `MediaSource` method the appended
/// text was reachable only from inside the "this message has blobs" branch, so a
/// message with something to say and nothing to inline rendered nothing.
#[tokio::test]
async fn extras_reach_a_message_with_no_media_at_all() {
let (store, frame) = store_and_frame("p14").await;
store
.append(frame, NewMessage::user("where am I").with_metadata(json!({ "extra": "files" })))
.await
.unwrap();
// No media hook at all: extras must not depend on one being registered.
let msgs = LinearAssembler::new()
.with_extras(Arc::new(Extras))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
.await
.unwrap();
assert_eq!(msgs[1], json!({ "role": "user", "content": "where am I\n[extra: files]" }));
}
#[tokio::test]
async fn one_appended_chunk_carries_both_halves_media_first() {
let (store, frame) = store_and_frame("p15").await;
store
.append(frame, NewMessage::user("look").with_metadata(json!({ "extra": "files" })))
.await
.unwrap();
// No vision ⇒ the image is skipped, so both halves have something to say.
let msgs = LinearAssembler::new()
.with_media(Arc::new(Media))
.with_extras(Arc::new(Extras))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
.await
.unwrap();
assert_eq!(msgs[1], json!({
"role": "user",
"content": "look\n[files: 1]\n[extra: files]",
}));
}
#[tokio::test]
async fn extras_see_the_previous_user_message_not_the_assistant_turn() {
let (store, frame) = store_and_frame("p16").await;
let meta = |v: &str| json!({ "extra": v });
store.append(frame, NewMessage::user("one").with_metadata(meta("files"))).await.unwrap();
store.append(frame, NewMessage::assistant("ok", None)).await.unwrap();
// Same view as the message before it, across an assistant turn: suppressed.
store.append(frame, NewMessage::user("two").with_metadata(meta("files"))).await.unwrap();
store.append(frame, NewMessage::assistant("ok", None)).await.unwrap();
// Changed view: emitted again.
store.append(frame, NewMessage::user("three").with_metadata(meta("projects"))).await.unwrap();
let msgs = LinearAssembler::new()
.with_extras(Arc::new(Extras))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
.await
.unwrap();
assert_eq!(msgs[1]["content"], "one\n[extra: files]", "prev = None ⇒ emitted");
assert_eq!(msgs[3]["content"], "two", "same as the previous user message ⇒ suppressed");
assert_eq!(msgs[5]["content"], "three\n[extra: projects]", "changed ⇒ emitted");
}
/// The parity contract: with no extras hook the output is what it always was.
#[tokio::test]
async fn no_extras_hook_changes_nothing() {
let (store, frame) = store_and_frame("p17").await;
store
.append(frame, NewMessage::user("look").with_metadata(json!({ "extra": "files" })))
.await
.unwrap();
let msgs = LinearAssembler::new()
.with_media(Arc::new(Media))
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
.await
.unwrap();
assert_eq!(msgs[1], json!({ "role": "user", "content": "look" }));
}