feat(chat): view context — tell the assistant what you're looking at
Nightly Build / build (push) Successful in 7m51s
Nightly Build / build (push) Successful in 7m51s
An eye next to the paperclip shares what the user has open with their next
message: the page, the folder being browsed, the file open in the viewer and
any highlighted passage (line numbers where a source view exists), plus which
entity a detail page is about. The bag is client-authored {label, value} pairs
in English — the backend only clamps (chars, never bytes), neutralizes the
harness tag and renders one <system-extra> block per message, deduped
consecutively so it appears exactly when the view changed. On by default,
per-device toggle, hover/tap to preview, a chip on every sent message;
docs/view-context.md for users, an updated harness.md clause for the model.
This commit is contained in:
@@ -15,7 +15,7 @@ use crate::activation::ActivationSource;
|
||||
use crate::ids::{ConversationId, FrameId};
|
||||
use crate::model::ModelInfo;
|
||||
use crate::projection::{
|
||||
MediaSource, Projection, ProjectionHooks, ResultLimit, ToolResultDigest,
|
||||
MediaSource, MessageExtras, Projection, ProjectionHooks, ResultLimit, ToolResultDigest,
|
||||
};
|
||||
use crate::store::HistoryStore;
|
||||
|
||||
@@ -157,6 +157,13 @@ impl LinearAssembler {
|
||||
self
|
||||
}
|
||||
|
||||
/// Text appended to each user/agent message (skipped media paths, the view
|
||||
/// the message was sent from…). One hook, one block — see [`MessageExtras`].
|
||||
pub fn with_extras(mut self, src: Arc<dyn MessageExtras>) -> Self {
|
||||
self.hooks.extras = Some(src);
|
||||
self
|
||||
}
|
||||
|
||||
/// How an over-long tool result is condensed.
|
||||
pub fn with_digest(mut self, digest: Arc<dyn ToolResultDigest>) -> Self {
|
||||
self.hooks.digest = Some(digest);
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
//!
|
||||
//! What the host owns: the **content** — the system prompt layers
|
||||
//! ([`crate::context::SystemContextSource`]), which media a message may inline
|
||||
//! ([`MediaSource`]) and how an over-long tool result is condensed
|
||||
//! ([`MediaSource`]), what extra text rides along with a message
|
||||
//! ([`MessageExtras`]) and how an over-long tool result is condensed
|
||||
//! ([`ToolResultDigest`]). Everything is optional: with no hooks at all the
|
||||
//! projection is a complete, correct OpenAI-shaped conversation.
|
||||
//!
|
||||
@@ -134,15 +135,36 @@ pub trait MediaSource: Send + Sync {
|
||||
async fn call_media(&self, _calls: &[StoredCall]) -> Vec<Arc<dyn MediaBlob>> {
|
||||
Vec::new()
|
||||
}
|
||||
/// Text appended to the message for the media that did NOT make it (a path
|
||||
/// list, so the agent can still reach them with a tool).
|
||||
}
|
||||
|
||||
/// Text appended to a user/agent message — the harness-generated tail a host
|
||||
/// wants the model to read alongside what the person typed (skipped attachment
|
||||
/// paths, the view the message was sent from, …).
|
||||
///
|
||||
/// **Its own hook, not a `MediaSource` method**, because it must run for every
|
||||
/// message, media or none: as a media method it was only ever reachable from
|
||||
/// inside the "this message has blobs" branch, so a message carrying nothing but
|
||||
/// non-media extras rendered nothing at all.
|
||||
///
|
||||
/// The crate does not wrap or frame what comes back — it appends the string
|
||||
/// verbatim, leading newlines included. Whatever block structure the host wants
|
||||
/// (`<system-extra>`…) is the host's, which is also why there is exactly **one**
|
||||
/// call per message: two hooks would mean two blocks.
|
||||
#[async_trait]
|
||||
pub trait MessageExtras: Send + Sync {
|
||||
/// `msg` is the message being projected; `prev` is the previous `User`/`Agent`
|
||||
/// message of the projected history (`None` for the first one, and after a
|
||||
/// compaction or a window cut), which lets a host suppress a repeat.
|
||||
///
|
||||
/// `skipped` are **positions in the vector `message_media` just returned**
|
||||
/// for this message, so the host can map them back to whatever it built
|
||||
/// them from.
|
||||
fn skipped_text(&self, _msg: &StoredMessage, _skipped: &[usize]) -> Option<String> {
|
||||
None
|
||||
}
|
||||
/// `skipped` are **positions in the vector [`MediaSource::message_media`]
|
||||
/// returned** for this message — empty when the message has no media at all,
|
||||
/// so a host must not read it as "nothing was left out of a media message".
|
||||
async fn appended_text(
|
||||
&self,
|
||||
msg: &StoredMessage,
|
||||
prev: Option<&StoredMessage>,
|
||||
skipped: &[usize],
|
||||
) -> Option<String>;
|
||||
}
|
||||
|
||||
/// How an over-long tool result is condensed. The crate decides *when*
|
||||
@@ -159,6 +181,7 @@ pub trait ToolResultDigest: Send + Sync {
|
||||
pub struct ProjectionHooks {
|
||||
pub activation: Option<Arc<dyn ActivationSource>>,
|
||||
pub media: Option<Arc<dyn MediaSource>>,
|
||||
pub extras: Option<Arc<dyn MessageExtras>>,
|
||||
pub digest: Option<Arc<dyn ToolResultDigest>>,
|
||||
}
|
||||
|
||||
@@ -213,10 +236,18 @@ pub async fn project(
|
||||
window(&mut history, max);
|
||||
}
|
||||
|
||||
// 4. The conversation.
|
||||
// 4. The conversation. `prev` trails one message behind so `MessageExtras`
|
||||
// can compare a message with the last thing the person said — carried as a
|
||||
// running reference rather than an `rposition` per message (same answer,
|
||||
// linear) and deliberately not put on `HistoryCtx`, which would drag a
|
||||
// `&[StoredMessage]` lifetime through the whole type for nothing.
|
||||
let ctx = HistoryCtx::new(&history, cfg, hooks, input).await?;
|
||||
let mut prev: Option<&StoredMessage> = None;
|
||||
for (idx, entry) in history.iter().enumerate() {
|
||||
ctx.project_message(&mut out, idx, entry).await;
|
||||
ctx.project_message(&mut out, idx, entry, prev).await;
|
||||
if matches!(entry.role, Role::User | Role::Agent) {
|
||||
prev = Some(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Dynamic tail — the fresh layers, as ONE trailing system message so a
|
||||
@@ -313,37 +344,57 @@ impl<'a> HistoryCtx<'a> {
|
||||
})
|
||||
}
|
||||
|
||||
async fn project_message(&self, out: &mut Vec<Value>, idx: usize, entry: &StoredMessage) {
|
||||
async fn project_message(
|
||||
&self,
|
||||
out: &mut Vec<Value>,
|
||||
idx: usize,
|
||||
entry: &StoredMessage,
|
||||
prev: Option<&StoredMessage>,
|
||||
) {
|
||||
match entry.role {
|
||||
// System messages are BUILT (layers 1-2), never replayed from the
|
||||
// store; a host that stores them gets them back verbatim.
|
||||
Role::System => out.push(json!({ "role": "system", "content": entry.content })),
|
||||
Role::User | Role::Agent => self.push_user(out, idx, entry).await,
|
||||
Role::User | Role::Agent => self.push_user(out, idx, entry, prev).await,
|
||||
Role::Assistant => self.push_assistant(out, idx, entry).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// A user/agent message: text plus, for the current turn, inlined media.
|
||||
async fn push_user(&self, out: &mut Vec<Value>, idx: usize, entry: &StoredMessage) {
|
||||
/// A user/agent message: text, the host's appended extras, and — for the
|
||||
/// current turn — inlined media.
|
||||
async fn push_user(
|
||||
&self,
|
||||
out: &mut Vec<Value>,
|
||||
idx: usize,
|
||||
entry: &StoredMessage,
|
||||
prev: Option<&StoredMessage>,
|
||||
) {
|
||||
let mut text = entry.content.clone();
|
||||
let mut parts: Vec<Value> = Vec::new();
|
||||
let mut skipped: Vec<usize> = Vec::new();
|
||||
|
||||
if let Some(src) = &self.hooks.media {
|
||||
let blobs = src.message_media(entry).await;
|
||||
if !blobs.is_empty() {
|
||||
// Older turns keep the textual path: everything is "skipped".
|
||||
let (inlined, skipped) = if idx >= self.media_turn_start {
|
||||
let (inlined, left_out) = if idx >= self.media_turn_start {
|
||||
media::partition(&blobs, &self.model.capabilities, &self.cfg.media).await
|
||||
} else {
|
||||
(Vec::new(), (0..blobs.len()).collect())
|
||||
};
|
||||
if let Some(extra) = src.skipped_text(entry, &skipped) {
|
||||
text.push_str(&extra);
|
||||
}
|
||||
skipped = left_out;
|
||||
parts = inlined;
|
||||
}
|
||||
}
|
||||
|
||||
// Outside the media branch on purpose: extras are not a media feature,
|
||||
// and a message with none must still get its block.
|
||||
if let Some(x) = &self.hooks.extras
|
||||
&& let Some(extra) = x.appended_text(entry, prev, &skipped).await
|
||||
{
|
||||
text.push_str(&extra);
|
||||
}
|
||||
|
||||
push_user_chunk(out, text, parts);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user