feat(chat): view context — tell the assistant what you're looking at
Nightly Build / build (push) Successful in 7m51s
Nightly Build / build (push) Successful in 7m51s
An eye next to the paperclip shares what the user has open with their next
message: the page, the folder being browsed, the file open in the viewer and
any highlighted passage (line numbers where a source view exists), plus which
entity a detail page is about. The bag is client-authored {label, value} pairs
in English — the backend only clamps (chars, never bytes), neutralizes the
harness tag and renders one <system-extra> block per message, deduped
consecutively so it appears exactly when the view changed. On by default,
per-device toggle, hover/tap to preview, a chip on every sent message;
docs/view-context.md for users, an updated harness.md clause for the model.
This commit is contained in:
@@ -10,7 +10,8 @@ use agent_loop::ids::{ConversationId, FrameId, MessageId};
|
||||
use agent_loop::model::ModelInfo;
|
||||
use agent_loop::prelude::async_trait;
|
||||
use agent_loop::projection::{
|
||||
MediaBlob, MediaSource, Projection, ReasoningEcho, ResultLimit, ToolResultDigest,
|
||||
MediaBlob, MediaSource, MessageExtras, Projection, ReasoningEcho, ResultLimit,
|
||||
ToolResultDigest,
|
||||
};
|
||||
use agent_loop::store::{
|
||||
CallOutcome, FrameSpec, HistoryStore, NewCall, NewMessage, NewSummary, StoredCall,
|
||||
@@ -432,8 +433,34 @@ impl MediaSource for Media {
|
||||
async fn call_media(&self, _calls: &[StoredCall]) -> Vec<Arc<dyn MediaBlob>> {
|
||||
vec![Arc::new(Png("tool.png"))]
|
||||
}
|
||||
fn skipped_text(&self, _msg: &StoredMessage, skipped: &[usize]) -> Option<String> {
|
||||
(!skipped.is_empty()).then(|| format!("\n[files: {}]", skipped.len()))
|
||||
}
|
||||
|
||||
/// The appended-text hook, in its own object: a note for the media left out, and
|
||||
/// — whatever the media — the message's `extra` metadata key, so the tests can
|
||||
/// tell "there was nothing to inline" from "there was nothing to say".
|
||||
struct Extras;
|
||||
|
||||
#[async_trait]
|
||||
impl MessageExtras for Extras {
|
||||
async fn appended_text(
|
||||
&self,
|
||||
msg: &StoredMessage,
|
||||
prev: Option<&StoredMessage>,
|
||||
skipped: &[usize],
|
||||
) -> Option<String> {
|
||||
let mut out = String::new();
|
||||
if !skipped.is_empty() {
|
||||
out.push_str(&format!("\n[files: {}]", skipped.len()));
|
||||
}
|
||||
let extra = |m: &StoredMessage| {
|
||||
m.metadata.as_ref().and_then(|v| v["extra"].as_str().map(str::to_string))
|
||||
};
|
||||
if let Some(e) = extra(msg)
|
||||
&& prev.and_then(extra) != Some(e.clone())
|
||||
{
|
||||
out.push_str(&format!("\n[extra: {e}]"));
|
||||
}
|
||||
(!out.is_empty()).then_some(out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -446,6 +473,7 @@ async fn media_is_inlined_for_the_current_turn_and_textual_before_it() {
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.with_extras(Arc::new(Extras))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
|
||||
capabilities: vec!["vision".into()],
|
||||
..ModelInfo::default()
|
||||
@@ -473,6 +501,7 @@ async fn a_model_without_vision_never_receives_bytes() {
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.with_extras(Arc::new(Extras))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -493,6 +522,7 @@ async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.with_extras(Arc::new(Extras))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
|
||||
capabilities: vec!["vision".into()],
|
||||
..ModelInfo::default()
|
||||
@@ -505,3 +535,89 @@ async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
|
||||
assert_eq!(last["content"][0]["type"], "image_url");
|
||||
assert_eq!(msgs[msgs.len() - 2]["role"], "tool", "it follows the result group");
|
||||
}
|
||||
|
||||
// ── Appended extras ──────────────────────────────────────────────────────────
|
||||
|
||||
/// The regression this hook exists for: as a `MediaSource` method the appended
|
||||
/// text was reachable only from inside the "this message has blobs" branch, so a
|
||||
/// message with something to say and nothing to inline rendered nothing.
|
||||
#[tokio::test]
|
||||
async fn extras_reach_a_message_with_no_media_at_all() {
|
||||
let (store, frame) = store_and_frame("p14").await;
|
||||
store
|
||||
.append(frame, NewMessage::user("where am I").with_metadata(json!({ "extra": "files" })))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// No media hook at all: extras must not depend on one being registered.
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_extras(Arc::new(Extras))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(msgs[1], json!({ "role": "user", "content": "where am I\n[extra: files]" }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_appended_chunk_carries_both_halves_media_first() {
|
||||
let (store, frame) = store_and_frame("p15").await;
|
||||
store
|
||||
.append(frame, NewMessage::user("look").with_metadata(json!({ "extra": "files" })))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// No vision ⇒ the image is skipped, so both halves have something to say.
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.with_extras(Arc::new(Extras))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(msgs[1], json!({
|
||||
"role": "user",
|
||||
"content": "look\n[files: 1]\n[extra: files]",
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn extras_see_the_previous_user_message_not_the_assistant_turn() {
|
||||
let (store, frame) = store_and_frame("p16").await;
|
||||
let meta = |v: &str| json!({ "extra": v });
|
||||
store.append(frame, NewMessage::user("one").with_metadata(meta("files"))).await.unwrap();
|
||||
store.append(frame, NewMessage::assistant("ok", None)).await.unwrap();
|
||||
// Same view as the message before it, across an assistant turn: suppressed.
|
||||
store.append(frame, NewMessage::user("two").with_metadata(meta("files"))).await.unwrap();
|
||||
store.append(frame, NewMessage::assistant("ok", None)).await.unwrap();
|
||||
// Changed view: emitted again.
|
||||
store.append(frame, NewMessage::user("three").with_metadata(meta("projects"))).await.unwrap();
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_extras(Arc::new(Extras))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(msgs[1]["content"], "one\n[extra: files]", "prev = None ⇒ emitted");
|
||||
assert_eq!(msgs[3]["content"], "two", "same as the previous user message ⇒ suppressed");
|
||||
assert_eq!(msgs[5]["content"], "three\n[extra: projects]", "changed ⇒ emitted");
|
||||
}
|
||||
|
||||
/// The parity contract: with no extras hook the output is what it always was.
|
||||
#[tokio::test]
|
||||
async fn no_extras_hook_changes_nothing() {
|
||||
let (store, frame) = store_and_frame("p17").await;
|
||||
store
|
||||
.append(frame, NewMessage::user("look").with_metadata(json!({ "extra": "files" })))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(msgs[1], json!({ "role": "user", "content": "look" }));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user