feat(chat): view context — tell the assistant what you're looking at

An eye next to the paperclip shares what the user has open with their next
message: the page, the folder being browsed, the file open in the viewer and
any highlighted passage (line numbers where a source view exists), plus which
entity a detail page is about. The bag is client-authored {label, value} pairs
in English — the backend only clamps (chars, never bytes), neutralizes the
harness tag and renders one <system-extra> block per message, deduped
consecutively so it appears exactly when the view changed. On by default,
per-device toggle, hover/tap to preview, a chip on every sent message;
docs/view-context.md for users, an updated harness.md clause for the model.

crates/agent-loop/src/context.rs

@@ -15,7 +15,7 @@ use crate::activation::ActivationSource;
 use crate::ids::{ConversationId, FrameId};
 use crate::model::ModelInfo;
 use crate::projection::{
-    MediaSource, Projection, ProjectionHooks, ResultLimit, ToolResultDigest,
+    MediaSource, MessageExtras, Projection, ProjectionHooks, ResultLimit, ToolResultDigest,
 };
 use crate::store::HistoryStore;
 
@@ -157,6 +157,13 @@ impl LinearAssembler {
         self
     }
 
+    /// Text appended to each user/agent message (skipped media paths, the view
+    /// the message was sent from…). One hook, one block — see [`MessageExtras`].
+    pub fn with_extras(mut self, src: Arc<dyn MessageExtras>) -> Self {
+        self.hooks.extras = Some(src);
+        self
+    }
+
     /// How an over-long tool result is condensed.
     pub fn with_digest(mut self, digest: Arc<dyn ToolResultDigest>) -> Self {
         self.hooks.digest = Some(digest);

crates/agent-loop/src/projection/mod.rs

@@ -9,7 +9,8 @@
 //!
 //! What the host owns: the **content** — the system prompt layers
 //! ([`crate::context::SystemContextSource`]), which media a message may inline
-//! ([`MediaSource`]) and how an over-long tool result is condensed
+//! ([`MediaSource`]), what extra text rides along with a message
+//! ([`MessageExtras`]) and how an over-long tool result is condensed
 //! ([`ToolResultDigest`]). Everything is optional: with no hooks at all the
 //! projection is a complete, correct OpenAI-shaped conversation.
 //!
@@ -134,15 +135,36 @@ pub trait MediaSource: Send + Sync {
     async fn call_media(&self, _calls: &[StoredCall]) -> Vec<Arc<dyn MediaBlob>> {
         Vec::new()
     }
-    /// Text appended to the message for the media that did NOT make it (a path
-    /// list, so the agent can still reach them with a tool).
+}
+
+/// Text appended to a user/agent message — the harness-generated tail a host
+/// wants the model to read alongside what the person typed (skipped attachment
+/// paths, the view the message was sent from, …).
+///
+/// **Its own hook, not a `MediaSource` method**, because it must run for every
+/// message, media or none: as a media method it was only ever reachable from
+/// inside the "this message has blobs" branch, so a message carrying nothing but
+/// non-media extras rendered nothing at all.
+///
+/// The crate does not wrap or frame what comes back — it appends the string
+/// verbatim, leading newlines included. Whatever block structure the host wants
+/// (`<system-extra>`…) is the host's, which is also why there is exactly **one**
+/// call per message: two hooks would mean two blocks.
+#[async_trait]
+pub trait MessageExtras: Send + Sync {
+    /// `msg` is the message being projected; `prev` is the previous `User`/`Agent`
+    /// message of the projected history (`None` for the first one, and after a
+    /// compaction or a window cut), which lets a host suppress a repeat.
     ///
-    /// `skipped` are **positions in the vector `message_media` just returned**
-    /// for this message, so the host can map them back to whatever it built
-    /// them from.
-    fn skipped_text(&self, _msg: &StoredMessage, _skipped: &[usize]) -> Option<String> {
-        None
-    }
+    /// `skipped` are **positions in the vector [`MediaSource::message_media`]
+    /// returned** for this message — empty when the message has no media at all,
+    /// so a host must not read it as "nothing was left out of a media message".
+    async fn appended_text(
+        &self,
+        msg:     &StoredMessage,
+        prev:    Option<&StoredMessage>,
+        skipped: &[usize],
+    ) -> Option<String>;
 }
 
 /// How an over-long tool result is condensed. The crate decides *when*
@@ -159,6 +181,7 @@ pub trait ToolResultDigest: Send + Sync {
 pub struct ProjectionHooks {
     pub activation: Option<Arc<dyn ActivationSource>>,
     pub media:      Option<Arc<dyn MediaSource>>,
+    pub extras:     Option<Arc<dyn MessageExtras>>,
     pub digest:     Option<Arc<dyn ToolResultDigest>>,
 }
 
@@ -213,10 +236,18 @@ pub async fn project(
         window(&mut history, max);
     }
 
-    // 4. The conversation.
+    // 4. The conversation. `prev` trails one message behind so `MessageExtras`
+    //    can compare a message with the last thing the person said — carried as a
+    //    running reference rather than an `rposition` per message (same answer,
+    //    linear) and deliberately not put on `HistoryCtx`, which would drag a
+    //    `&[StoredMessage]` lifetime through the whole type for nothing.
     let ctx = HistoryCtx::new(&history, cfg, hooks, input).await?;
+    let mut prev: Option<&StoredMessage> = None;
     for (idx, entry) in history.iter().enumerate() {
-        ctx.project_message(&mut out, idx, entry).await;
+        ctx.project_message(&mut out, idx, entry, prev).await;
+        if matches!(entry.role, Role::User | Role::Agent) {
+            prev = Some(entry);
+        }
     }
 
     // 5. Dynamic tail — the fresh layers, as ONE trailing system message so a
@@ -313,37 +344,57 @@ impl<'a> HistoryCtx<'a> {
         })
     }
 
-    async fn project_message(&self, out: &mut Vec<Value>, idx: usize, entry: &StoredMessage) {
+    async fn project_message(
+        &self,
+        out:   &mut Vec<Value>,
+        idx:   usize,
+        entry: &StoredMessage,
+        prev:  Option<&StoredMessage>,
+    ) {
         match entry.role {
             // System messages are BUILT (layers 1-2), never replayed from the
             // store; a host that stores them gets them back verbatim.
             Role::System => out.push(json!({ "role": "system", "content": entry.content })),
-            Role::User | Role::Agent => self.push_user(out, idx, entry).await,
+            Role::User | Role::Agent => self.push_user(out, idx, entry, prev).await,
             Role::Assistant => self.push_assistant(out, idx, entry).await,
         }
     }
 
-    /// A user/agent message: text plus, for the current turn, inlined media.
-    async fn push_user(&self, out: &mut Vec<Value>, idx: usize, entry: &StoredMessage) {
+    /// A user/agent message: text, the host's appended extras, and — for the
+    /// current turn — inlined media.
+    async fn push_user(
+        &self,
+        out:   &mut Vec<Value>,
+        idx:   usize,
+        entry: &StoredMessage,
+        prev:  Option<&StoredMessage>,
+    ) {
         let mut text = entry.content.clone();
         let mut parts: Vec<Value> = Vec::new();
+        let mut skipped: Vec<usize> = Vec::new();
 
         if let Some(src) = &self.hooks.media {
             let blobs = src.message_media(entry).await;
             if !blobs.is_empty() {
                 // Older turns keep the textual path: everything is "skipped".
-                let (inlined, skipped) = if idx >= self.media_turn_start {
+                let (inlined, left_out) = if idx >= self.media_turn_start {
                     media::partition(&blobs, &self.model.capabilities, &self.cfg.media).await
                 } else {
                     (Vec::new(), (0..blobs.len()).collect())
                 };
-                if let Some(extra) = src.skipped_text(entry, &skipped) {
-                    text.push_str(&extra);
-                }
+                skipped = left_out;
                 parts = inlined;
             }
         }
 
+        // Outside the media branch on purpose: extras are not a media feature,
+        // and a message with none must still get its block.
+        if let Some(x) = &self.hooks.extras
+            && let Some(extra) = x.appended_text(entry, prev, &skipped).await
+        {
+            text.push_str(&extra);
+        }
+
         push_user_chunk(out, text, parts);
     }
 

crates/agent-loop/tests/projection.rs

@@ -10,7 +10,8 @@ use agent_loop::ids::{ConversationId, FrameId, MessageId};
 use agent_loop::model::ModelInfo;
 use agent_loop::prelude::async_trait;
 use agent_loop::projection::{
-    MediaBlob, MediaSource, Projection, ReasoningEcho, ResultLimit, ToolResultDigest,
+    MediaBlob, MediaSource, MessageExtras, Projection, ReasoningEcho, ResultLimit,
+    ToolResultDigest,
 };
 use agent_loop::store::{
     CallOutcome, FrameSpec, HistoryStore, NewCall, NewMessage, NewSummary, StoredCall,
@@ -432,8 +433,34 @@ impl MediaSource for Media {
     async fn call_media(&self, _calls: &[StoredCall]) -> Vec<Arc<dyn MediaBlob>> {
         vec![Arc::new(Png("tool.png"))]
     }
-    fn skipped_text(&self, _msg: &StoredMessage, skipped: &[usize]) -> Option<String> {
-        (!skipped.is_empty()).then(|| format!("\n[files: {}]", skipped.len()))
+}
+
+/// The appended-text hook, in its own object: a note for the media left out, and
+/// — whatever the media — the message's `extra` metadata key, so the tests can
+/// tell "there was nothing to inline" from "there was nothing to say".
+struct Extras;
+
+#[async_trait]
+impl MessageExtras for Extras {
+    async fn appended_text(
+        &self,
+        msg:     &StoredMessage,
+        prev:    Option<&StoredMessage>,
+        skipped: &[usize],
+    ) -> Option<String> {
+        let mut out = String::new();
+        if !skipped.is_empty() {
+            out.push_str(&format!("\n[files: {}]", skipped.len()));
+        }
+        let extra = |m: &StoredMessage| {
+            m.metadata.as_ref().and_then(|v| v["extra"].as_str().map(str::to_string))
+        };
+        if let Some(e) = extra(msg)
+            && prev.and_then(extra) != Some(e.clone())
+        {
+            out.push_str(&format!("\n[extra: {e}]"));
+        }
+        (!out.is_empty()).then_some(out)
     }
 }
 
@@ -446,6 +473,7 @@ async fn media_is_inlined_for_the_current_turn_and_textual_before_it() {
 
     let msgs = LinearAssembler::new()
         .with_media(Arc::new(Media))
+        .with_extras(Arc::new(Extras))
         .build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
             capabilities: vec!["vision".into()],
             ..ModelInfo::default()
@@ -473,6 +501,7 @@ async fn a_model_without_vision_never_receives_bytes() {
 
     let msgs = LinearAssembler::new()
         .with_media(Arc::new(Media))
+        .with_extras(Arc::new(Extras))
         .build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
         .await
         .unwrap();
@@ -493,6 +522,7 @@ async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
 
     let msgs = LinearAssembler::new()
         .with_media(Arc::new(Media))
+        .with_extras(Arc::new(Extras))
         .build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
             capabilities: vec!["vision".into()],
             ..ModelInfo::default()
@@ -505,3 +535,89 @@ async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
     assert_eq!(last["content"][0]["type"], "image_url");
     assert_eq!(msgs[msgs.len() - 2]["role"], "tool", "it follows the result group");
 }
+
+// ── Appended extras ──────────────────────────────────────────────────────────
+
+/// The regression this hook exists for: as a `MediaSource` method the appended
+/// text was reachable only from inside the "this message has blobs" branch, so a
+/// message with something to say and nothing to inline rendered nothing.
+#[tokio::test]
+async fn extras_reach_a_message_with_no_media_at_all() {
+    let (store, frame) = store_and_frame("p14").await;
+    store
+        .append(frame, NewMessage::user("where am I").with_metadata(json!({ "extra": "files" })))
+        .await
+        .unwrap();
+
+    // No media hook at all: extras must not depend on one being registered.
+    let msgs = LinearAssembler::new()
+        .with_extras(Arc::new(Extras))
+        .build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
+        .await
+        .unwrap();
+
+    assert_eq!(msgs[1], json!({ "role": "user", "content": "where am I\n[extra: files]" }));
+}
+
+#[tokio::test]
+async fn one_appended_chunk_carries_both_halves_media_first() {
+    let (store, frame) = store_and_frame("p15").await;
+    store
+        .append(frame, NewMessage::user("look").with_metadata(json!({ "extra": "files" })))
+        .await
+        .unwrap();
+
+    // No vision ⇒ the image is skipped, so both halves have something to say.
+    let msgs = LinearAssembler::new()
+        .with_media(Arc::new(Media))
+        .with_extras(Arc::new(Extras))
+        .build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
+        .await
+        .unwrap();
+
+    assert_eq!(msgs[1], json!({
+        "role": "user",
+        "content": "look\n[files: 1]\n[extra: files]",
+    }));
+}
+
+#[tokio::test]
+async fn extras_see_the_previous_user_message_not_the_assistant_turn() {
+    let (store, frame) = store_and_frame("p16").await;
+    let meta = |v: &str| json!({ "extra": v });
+    store.append(frame, NewMessage::user("one").with_metadata(meta("files"))).await.unwrap();
+    store.append(frame, NewMessage::assistant("ok", None)).await.unwrap();
+    // Same view as the message before it, across an assistant turn: suppressed.
+    store.append(frame, NewMessage::user("two").with_metadata(meta("files"))).await.unwrap();
+    store.append(frame, NewMessage::assistant("ok", None)).await.unwrap();
+    // Changed view: emitted again.
+    store.append(frame, NewMessage::user("three").with_metadata(meta("projects"))).await.unwrap();
+
+    let msgs = LinearAssembler::new()
+        .with_extras(Arc::new(Extras))
+        .build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
+        .await
+        .unwrap();
+
+    assert_eq!(msgs[1]["content"], "one\n[extra: files]", "prev = None ⇒ emitted");
+    assert_eq!(msgs[3]["content"], "two", "same as the previous user message ⇒ suppressed");
+    assert_eq!(msgs[5]["content"], "three\n[extra: projects]", "changed ⇒ emitted");
+}
+
+/// The parity contract: with no extras hook the output is what it always was.
+#[tokio::test]
+async fn no_extras_hook_changes_nothing() {
+    let (store, frame) = store_and_frame("p17").await;
+    store
+        .append(frame, NewMessage::user("look").with_metadata(json!({ "extra": "files" })))
+        .await
+        .unwrap();
+
+    let msgs = LinearAssembler::new()
+        .with_media(Arc::new(Media))
+        .build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
+        .await
+        .unwrap();
+
+    assert_eq!(msgs[1], json!({ "role": "user", "content": "look" }));
+}