agent-loop: projection, recovery, compaction into the crate (phase 3)
Nightly Build / build (push) Successful in 6m49s
Nightly Build / build (push) Successful in 6m49s
The session handler is now a thin shell: three entry points in kernel_turn.rs (run_kernel_turn / recover_turn / resolve_pending_call) and the ChatSessionHandler. Everything that shaped a Value — projection, recovery, compaction mechanics, the LLM loop, message building — lives in agent-loop or behind a loop_adapters trait. agent-loop: - projection/ (mod + media): stored history -> wire messages, the one place provider divergence lives; well-formedness contract, DTL injections (append-only), media parts. LinearAssembler is now a Projection + ProjectionHooks config, not its own implementation - recovery.rs: reap interrupted batches -> resolve the deepest frame's non-terminal calls (Running by policy + RestartHint, AwaitingHuman re-asked) -> un-wedge finished children -> cascade up, every frame on its own agent (B3) - compaction.rs: split point (never assistant+tool group), transcript, SUMMARY_PREFIX/preamble/template, the no-tools model call, summary row - manager: resolve_pending (gate skipped, real ToolContext, then continue incl. sub-agent); start_loop used by recovery; LiveInput - delegate: AsyncExecutor + StoreSink for mode:async (durable cron row, result delivered back into the parent conversation) - kernel/context/store: support the above (TurnScope via Extensions, frame lookups, aligned result-text semantics) skald-core: - loop_adapters: UserLoopRuntime (D12 - one LoopManager per user), TurnScope (per-turn state in the Extensions type-map; no scope is denied), projection_cfg/media_source/tool_digest (Skald's projection knobs without owning projection code), async_task (CronExecutor + DurableSink) - session/handler: stripped to mod.rs + kernel_turn.rs + config.rs + interface_tools.rs + media.rs; deleted agent_dispatch, approval, dispatch, emitter, gate, llm_call, llm_loop, message_builder, messages, outcome, resume - compactor.rs: policy only (threshold, model pick, CompactionEvent); mechanics are the crate's CLAUDE.md updated (recovery, compaction, sub-agents, approval gate, projection sections now describe the crate-owned flow).
This commit is contained in:
@@ -143,6 +143,7 @@ async fn params(
|
||||
system: Arc::new(StaticSystemContext::new("You are a test agent.")),
|
||||
tools,
|
||||
model_hint: ModelHint::default(),
|
||||
selector: None,
|
||||
live_input: None,
|
||||
extensions: Default::default(),
|
||||
meta: TurnMeta::default(),
|
||||
@@ -502,18 +503,26 @@ async fn second_loop_on_same_conversation_rejected() {
|
||||
|
||||
struct TestCatalog {
|
||||
context: Arc<StaticSystemContext>,
|
||||
/// Pins the child to its own model, so a test can script parent and child
|
||||
/// independently (a shared script would race on who pops which step).
|
||||
model: Option<ModelHint>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AgentCatalog for TestCatalog {
|
||||
async fn get(&self, id: &str, _child_frame: agent_loop::ids::FrameId) -> agent_loop::Result<AgentProfile> {
|
||||
async fn get(
|
||||
&self,
|
||||
id: &str,
|
||||
_child_frame: agent_loop::ids::FrameId,
|
||||
_ctx: &agent_loop::tool::ToolCtx,
|
||||
) -> agent_loop::Result<AgentProfile> {
|
||||
Ok(AgentProfile {
|
||||
id: id.into(),
|
||||
kind: AgentKind::Task,
|
||||
context: self.context.clone(),
|
||||
tools: ToolSelection::inherit(),
|
||||
toolset: None,
|
||||
model: None,
|
||||
model: self.model.clone(),
|
||||
selector: None,
|
||||
assembler: None,
|
||||
})
|
||||
@@ -540,6 +549,7 @@ async fn sync_delegate_runs_child_loop_and_returns_result() {
|
||||
);
|
||||
let catalog: Arc<dyn AgentCatalog> = Arc::new(TestCatalog {
|
||||
context: Arc::new(StaticSystemContext::new("You are a researcher.")),
|
||||
model: None,
|
||||
});
|
||||
let delegate: Arc<dyn Tool> = Arc::new(DelegateTool::new(manager.clone(), catalog, manager.store(), 5));
|
||||
|
||||
@@ -551,6 +561,7 @@ async fn sync_delegate_runs_child_loop_and_returns_result() {
|
||||
system: Arc::new(StaticSystemContext::new("root")),
|
||||
tools: ToolRegistry::new().with_arc(delegate).into_toolset(),
|
||||
model_hint: ModelHint::default(),
|
||||
selector: None,
|
||||
live_input: None,
|
||||
extensions: Default::default(),
|
||||
meta: TurnMeta::default(),
|
||||
@@ -599,6 +610,7 @@ async fn delegate_batch_fans_out_concurrently() {
|
||||
);
|
||||
let catalog: Arc<dyn AgentCatalog> = Arc::new(TestCatalog {
|
||||
context: Arc::new(StaticSystemContext::new("worker")),
|
||||
model: None,
|
||||
});
|
||||
let delegate: Arc<dyn Tool> = Arc::new(DelegateTool::new(manager.clone(), catalog, manager.store(), 5));
|
||||
|
||||
@@ -610,6 +622,7 @@ async fn delegate_batch_fans_out_concurrently() {
|
||||
system: Arc::new(StaticSystemContext::new("root")),
|
||||
tools: ToolRegistry::new().with_arc(delegate).into_toolset(),
|
||||
model_hint: ModelHint::default(),
|
||||
selector: None,
|
||||
live_input: None,
|
||||
extensions: Default::default(),
|
||||
meta: TurnMeta::default(),
|
||||
@@ -633,4 +646,178 @@ async fn delegate_batch_fans_out_concurrently() {
|
||||
);
|
||||
}
|
||||
|
||||
// ── async delegation ──
|
||||
|
||||
/// Polls until `f` holds, so a background delivery does not need a sleep.
|
||||
async fn eventually<F, Fut>(label: &str, f: F)
|
||||
where
|
||||
F: Fn() -> Fut,
|
||||
Fut: std::future::Future<Output = bool>,
|
||||
{
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
while std::time::Instant::now() < deadline {
|
||||
if f().await {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("timed out waiting for: {label}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_delegate_returns_a_receipt_then_delivers_the_result() {
|
||||
// Parent and child get their own scripted model: the parent does NOT wait
|
||||
// for the child, so one shared script would race on who pops which step.
|
||||
let root = Arc::new(FakeModel::new("root", vec![
|
||||
Step::tool_calls("", vec![testing::call("c1", "delegate", json!({
|
||||
"agent_id": "worker", "prompt": "long job", "mode": "async", "title": "nightly",
|
||||
}))]),
|
||||
Step::message("started it"),
|
||||
]));
|
||||
let child = Arc::new(FakeModel::new("child", vec![Step::message("the long answer")]));
|
||||
|
||||
let store = Arc::new(InMemoryStore::new());
|
||||
let manager = Arc::new(
|
||||
LoopManager::builder()
|
||||
.models(Arc::new(StaticModels::new(vec![
|
||||
testing::handle(&root, "root"),
|
||||
testing::handle(&child, "child"),
|
||||
])))
|
||||
.store(store.clone())
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
let catalog: Arc<dyn AgentCatalog> = Arc::new(TestCatalog {
|
||||
context: Arc::new(StaticSystemContext::new("worker")),
|
||||
model: Some(ModelHint::name("child")),
|
||||
});
|
||||
let sink: Arc<dyn AsyncResultSink> = Arc::new(StoreSink::new(manager.store()));
|
||||
let exec: Arc<dyn AsyncExecutor> = Arc::new(InProcessExecutor::new(
|
||||
manager.clone(),
|
||||
catalog.clone(),
|
||||
manager.store(),
|
||||
sink,
|
||||
ToolRegistry::new().into_toolset(),
|
||||
));
|
||||
let delegate: Arc<dyn Tool> = Arc::new(
|
||||
DelegateTool::new(manager.clone(), catalog, manager.store(), 5).with_async(exec),
|
||||
);
|
||||
|
||||
let conv = ConversationId::new("d3");
|
||||
let frame = manager.open_root(&conv, FrameSpec::root("assistant")).await.unwrap();
|
||||
let p = TurnParams {
|
||||
frame,
|
||||
agent: "assistant".into(),
|
||||
system: Arc::new(StaticSystemContext::new("root")),
|
||||
tools: ToolRegistry::new().with_arc(delegate).into_toolset(),
|
||||
model_hint: ModelHint::default(),
|
||||
selector: None,
|
||||
live_input: None,
|
||||
extensions: Default::default(),
|
||||
meta: TurnMeta::default(),
|
||||
assembler: None,
|
||||
};
|
||||
|
||||
let handle = manager.start_turn(conv.clone(), NewMessage::user("run it"), p).await.unwrap();
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(5), handle.join())
|
||||
.await
|
||||
.expect("async delegate must not block the parent turn")
|
||||
.unwrap();
|
||||
let TurnOutcome::Final { content, .. } = outcome else { panic!("got {outcome:?}") };
|
||||
assert_eq!(content, "started it");
|
||||
|
||||
// The delegating call resolved with a receipt, not with the child's answer.
|
||||
let done = store.calls_in_state(frame, &[CallState::Done]).await.unwrap();
|
||||
let receipt: Value =
|
||||
serde_json::from_str(done[0].result.as_deref().unwrap()).expect("receipt is JSON");
|
||||
assert_eq!(receipt["status"], "started");
|
||||
assert_eq!(receipt["task_id"], 1);
|
||||
|
||||
// …and the answer lands later, as its own completed call.
|
||||
let store_c = store.clone();
|
||||
eventually("the delivered result", || {
|
||||
let store = store_c.clone();
|
||||
async move {
|
||||
store
|
||||
.load(frame)
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|m| m.calls.iter().any(|c| c.name == agent_loop::delegate::DELIVERY_CALL))
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
let history = store.load(frame).await.unwrap();
|
||||
let delivery = history
|
||||
.iter()
|
||||
.find(|m| m.calls.iter().any(|c| c.name == agent_loop::delegate::DELIVERY_CALL))
|
||||
.unwrap();
|
||||
assert!(delivery.synthetic, "the delivery is not a turn the user drove");
|
||||
let call = &delivery.calls[0];
|
||||
assert_eq!(call.state, CallState::Done);
|
||||
let payload: Value = serde_json::from_str(call.result.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(payload["task_id"], 1);
|
||||
assert_eq!(payload["title"], "nightly");
|
||||
assert_eq!(payload["result"], "the long answer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_delegate_without_an_executor_is_refused() {
|
||||
let script = vec![
|
||||
Step::tool_calls("", vec![testing::call("c1", "delegate", json!({
|
||||
"agent_id": "worker", "prompt": "job", "mode": "async",
|
||||
}))]),
|
||||
Step::message("could not start it"),
|
||||
];
|
||||
let store = Arc::new(InMemoryStore::new());
|
||||
let manager = Arc::new(
|
||||
LoopManager::builder()
|
||||
.models(Arc::new(agent_loop::model::SingleModel::new(FakeModel::new("m", script))))
|
||||
.store(store.clone())
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
let catalog: Arc<dyn AgentCatalog> = Arc::new(TestCatalog {
|
||||
context: Arc::new(StaticSystemContext::new("worker")),
|
||||
model: None,
|
||||
});
|
||||
// No `with_async`: the mode must fail, never silently run sync — a turn
|
||||
// that asked not to wait would otherwise block on the child.
|
||||
let delegate: Arc<dyn Tool> =
|
||||
Arc::new(DelegateTool::new(manager.clone(), catalog, manager.store(), 5));
|
||||
|
||||
let conv = ConversationId::new("d4");
|
||||
let frame = manager.open_root(&conv, FrameSpec::root("assistant")).await.unwrap();
|
||||
let p = TurnParams {
|
||||
frame,
|
||||
agent: "assistant".into(),
|
||||
system: Arc::new(StaticSystemContext::new("root")),
|
||||
tools: ToolRegistry::new().with_arc(delegate).into_toolset(),
|
||||
model_hint: ModelHint::default(),
|
||||
selector: None,
|
||||
live_input: None,
|
||||
extensions: Default::default(),
|
||||
meta: TurnMeta::default(),
|
||||
assembler: None,
|
||||
};
|
||||
|
||||
let handle = manager.start_turn(conv.clone(), NewMessage::user("run it"), p).await.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), handle.join())
|
||||
.await
|
||||
.expect("turn hung")
|
||||
.unwrap();
|
||||
|
||||
let failed = store.calls_in_state(frame, &[CallState::Failed]).await.unwrap();
|
||||
assert_eq!(failed.len(), 1);
|
||||
assert!(
|
||||
failed[0].result.as_deref().unwrap().contains("async mode is not available"),
|
||||
"{:?}",
|
||||
failed[0].result
|
||||
);
|
||||
// Nothing was spawned: no child frame was ever opened.
|
||||
assert!(store.active_frames(&conv).await.unwrap().iter().all(|f| f.spec.depth == 0));
|
||||
}
|
||||
|
||||
use agent_loop::delegate::{AsyncExecutor, AsyncResultSink, InProcessExecutor, StoreSink};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -0,0 +1,507 @@
|
||||
//! Golden tests of the projection (blueprint §13): the exact wire shape of
|
||||
//! every layer, for every provider knob. These assert full messages, not just
|
||||
//! properties — a change in what a model receives must show up here.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use agent_loop::activation::{Activation, ActivationSource, ToolRendering};
|
||||
use agent_loop::context::{AssembleInput, ContextAssembler, LinearAssembler, SystemContext};
|
||||
use agent_loop::ids::{ConversationId, FrameId, MessageId};
|
||||
use agent_loop::model::ModelInfo;
|
||||
use agent_loop::prelude::async_trait;
|
||||
use agent_loop::projection::{
|
||||
MediaBlob, MediaSource, Projection, ReasoningEcho, ResultLimit, ToolResultDigest,
|
||||
};
|
||||
use agent_loop::store::{
|
||||
CallOutcome, FrameSpec, HistoryStore, NewCall, NewMessage, NewSummary, StoredCall,
|
||||
StoredMessage,
|
||||
};
|
||||
use agent_loop::store_memory::InMemoryStore;
|
||||
use agent_loop::tool::ToolOutput;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
// ── fixtures ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async fn store_and_frame(name: &str) -> (Arc<dyn HistoryStore>, FrameId) {
|
||||
let store = Arc::new(InMemoryStore::new());
|
||||
let conv = ConversationId::new(name);
|
||||
let frame = store.open_frame(&conv, None, FrameSpec::root("assistant")).await.unwrap();
|
||||
(store, frame)
|
||||
}
|
||||
|
||||
fn input(frame: FrameId, system: SystemContext, model: ModelInfo) -> AssembleInput {
|
||||
AssembleInput { frame, system, model, round: 0 }
|
||||
}
|
||||
|
||||
fn tool_def(name: &str) -> Value {
|
||||
json!({"type":"function","function":{"name":name,"parameters":{"type":"object"}}})
|
||||
}
|
||||
|
||||
/// The Skald-flavoured configuration: every knob off the default, so the test
|
||||
/// exercises the parameterization rather than the defaults.
|
||||
fn strict() -> Projection {
|
||||
Projection {
|
||||
summary_suffix: Some("[End of summary]".into()),
|
||||
interrupted_text: "Error: tool call was interrupted.".into(),
|
||||
rejected_default: "User rejected this tool call.".into(),
|
||||
cancelled_default: "Tool call was cancelled by the user.".into(),
|
||||
reasoning_placeholder: Some("(no reasoning recorded for this step)".into()),
|
||||
reasoning_echo: ReasoningEcho::Both,
|
||||
activation_anchor_tool: Some("activate_tools".into()),
|
||||
..Projection::default()
|
||||
}
|
||||
}
|
||||
|
||||
struct Stub(Vec<Activation>);
|
||||
|
||||
#[async_trait]
|
||||
impl ActivationSource for Stub {
|
||||
async fn activations(&self, _frame: FrameId) -> agent_loop::Result<Vec<Activation>> {
|
||||
Ok(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
// ── system layers ────────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn prompt_cache_turns_the_static_prefix_into_a_cache_breakpoint() {
|
||||
let (store, frame) = store_and_frame("p1").await;
|
||||
|
||||
let plain = LinearAssembler::new()
|
||||
.build(&store, &input(frame, SystemContext::base("BASE"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(plain[0], json!({ "role": "system", "content": "BASE" }));
|
||||
|
||||
let cached = LinearAssembler::new()
|
||||
.build(&store, &input(frame, SystemContext::base("BASE"), ModelInfo {
|
||||
prompt_cache: true,
|
||||
..ModelInfo::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
cached[0],
|
||||
json!({
|
||||
"role": "system",
|
||||
"content": [{ "type": "text", "text": "BASE",
|
||||
"cache_control": { "type": "ephemeral" } }],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn static_and_dynamic_layers_land_on_their_sides_of_the_history() {
|
||||
let (store, frame) = store_and_frame("p2").await;
|
||||
store.append(frame, NewMessage::user("hi")).await.unwrap();
|
||||
|
||||
let system = SystemContext::base("BASE")
|
||||
.with_static("FORMAT RULES")
|
||||
.with_static("<scratchpad/>")
|
||||
.with_dynamic("MEMORY")
|
||||
.with_dynamic("NOW")
|
||||
.with_reminder("REMEMBER");
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.build(&store, &input(frame, system, ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(msgs, vec![
|
||||
json!({ "role": "system", "content": "BASE" }),
|
||||
json!({ "role": "system", "content": "FORMAT RULES" }),
|
||||
json!({ "role": "system", "content": "<scratchpad/>" }),
|
||||
json!({ "role": "user", "content": "hi" }),
|
||||
// The dynamic layers are ONE trailing block, joined by the separator.
|
||||
json!({ "role": "system", "content": "MEMORY\n\n---\nNOW" }),
|
||||
json!({ "role": "system", "content": "REMEMBER" }),
|
||||
]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn summary_replaces_covered_history_and_carries_its_suffix() {
|
||||
let (store, frame) = store_and_frame("p3").await;
|
||||
let m1 = store.append(frame, NewMessage::user("old question")).await.unwrap();
|
||||
store.append(frame, NewMessage::assistant("old answer", None)).await.unwrap();
|
||||
store.append(frame, NewMessage::user("new question")).await.unwrap();
|
||||
store
|
||||
.save_summary(frame, NewSummary { text: "They discussed old stuff.".into(), covered_up_to: m1 })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(strict())
|
||||
.build(&store, &input(frame, SystemContext::base("BASE"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
msgs[1],
|
||||
json!({
|
||||
"role": "system",
|
||||
"content": "[CONTEXT SUMMARY — earlier messages were compacted into this summary]\n\n\
|
||||
They discussed old stuff.\n\n[End of summary]",
|
||||
})
|
||||
);
|
||||
let joined = msgs.iter().filter_map(|m| m["content"].as_str()).collect::<Vec<_>>().join("|");
|
||||
assert!(joined.contains("old answer"), "history after the cut must survive");
|
||||
assert!(!joined.contains("old question"), "covered history must be gone");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_window_never_opens_on_half_an_exchange() {
|
||||
let (store, frame) = store_and_frame("p4").await;
|
||||
store.append(frame, NewMessage::user("first")).await.unwrap();
|
||||
let asst = store.append(frame, NewMessage::assistant("calling", None)).await.unwrap();
|
||||
let call = store.append_call(asst, NewCall::new("t", json!({})).with_provider_id("c1")).await.unwrap();
|
||||
store.resolve_call(call, &CallOutcome::Completed(ToolOutput::Text("r".into()))).await.unwrap();
|
||||
store.append(frame, NewMessage::user("second")).await.unwrap();
|
||||
|
||||
// A window of 2 would start on the assistant+tool group: it is dropped.
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_max_messages(2)
|
||||
.build(&store, &input(frame, SystemContext::base("BASE"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let roles: Vec<&str> = msgs.iter().filter_map(|m| m["role"].as_str()).collect();
|
||||
assert_eq!(roles, ["system", "user"]);
|
||||
}
|
||||
|
||||
// ── tool calls and results ───────────────────────────────────────────────────
|
||||
|
||||
/// Seeds one assistant turn with a call in each terminal state, plus a survivor.
|
||||
async fn seed_states(store: &Arc<dyn HistoryStore>, frame: FrameId) -> MessageId {
|
||||
store.append(frame, NewMessage::user("go")).await.unwrap();
|
||||
let msg = store.append(frame, NewMessage::assistant("working", None)).await.unwrap();
|
||||
|
||||
let done = store.append_call(msg, NewCall::new("a", json!({})).with_provider_id("c1")).await.unwrap();
|
||||
store.resolve_call(done, &CallOutcome::Completed(ToolOutput::Text("ok".into()))).await.unwrap();
|
||||
|
||||
let failed = store.append_call(msg, NewCall::new("b", json!({})).with_provider_id("c2")).await.unwrap();
|
||||
store.resolve_call(failed, &CallOutcome::Failed("boom".into())).await.unwrap();
|
||||
|
||||
let rejected = store.append_call(msg, NewCall::new("c", json!({})).with_provider_id("c3")).await.unwrap();
|
||||
store.resolve_call(rejected, &CallOutcome::Rejected { reason: String::new() }).await.unwrap();
|
||||
|
||||
let cancelled = store.append_call(msg, NewCall::new("d", json!({})).with_provider_id("c4")).await.unwrap();
|
||||
store.resolve_call(cancelled, &CallOutcome::Cancelled).await.unwrap();
|
||||
|
||||
// Never resolved: a crash survivor.
|
||||
store.append_call(msg, NewCall::new("e", json!({})).with_provider_id("c5")).await.unwrap();
|
||||
msg
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_call_state_gets_a_result_the_model_can_read() {
|
||||
let (store, frame) = store_and_frame("p5").await;
|
||||
seed_states(&store, frame).await;
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(strict())
|
||||
.build(&store, &input(frame, SystemContext::base("BASE"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let results: Vec<(&str, &str)> = msgs
|
||||
.iter()
|
||||
.filter(|m| m["role"] == "tool")
|
||||
.map(|m| (m["tool_call_id"].as_str().unwrap(), m["content"].as_str().unwrap()))
|
||||
.collect();
|
||||
assert_eq!(results, vec![
|
||||
("c1", "ok"),
|
||||
("c2", "Error: boom"),
|
||||
// The rejection recorded an empty reason: the configured note stands in.
|
||||
("c3", "User rejected this tool call."),
|
||||
// A recorded note wins over the configured default.
|
||||
("c4", "Cancelled by user."),
|
||||
("c5", "Error: tool call was interrupted."),
|
||||
]);
|
||||
|
||||
// The assistant turn itself: calls in order, and a stand-in reasoning
|
||||
// because none was recorded.
|
||||
let asst = msgs.iter().find(|m| m["role"] == "assistant").unwrap();
|
||||
assert_eq!(asst["tool_calls"][0], json!({
|
||||
"id": "c1", "type": "function",
|
||||
"function": { "name": "a", "arguments": "{}" },
|
||||
}));
|
||||
assert_eq!(asst["reasoning_content"], "(no reasoning recorded for this step)");
|
||||
assert_eq!(asst["reasoning"], "(no reasoning recorded for this step)");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_echo_is_per_provider_and_never_empty() {
|
||||
let (store, frame) = store_and_frame("p6").await;
|
||||
store.append(frame, NewMessage::user("q")).await.unwrap();
|
||||
store.append(frame, NewMessage::assistant("a", Some("because".into()))).await.unwrap();
|
||||
store.append(frame, NewMessage::user("q2")).await.unwrap();
|
||||
// An empty stored reasoning must not produce an empty field.
|
||||
store.append(frame, NewMessage::assistant("a2", Some(String::new()))).await.unwrap();
|
||||
|
||||
let one = LinearAssembler::new()
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = one.iter().find(|m| m["content"] == "a").unwrap();
|
||||
assert_eq!(first["reasoning_content"], "because");
|
||||
assert!(first.get("reasoning").is_none(), "ContentOnly must not echo `reasoning`");
|
||||
let second = one.iter().find(|m| m["content"] == "a2").unwrap();
|
||||
assert!(second.get("reasoning_content").is_none());
|
||||
|
||||
let both = LinearAssembler::new()
|
||||
.with_projection(strict())
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = both.iter().find(|m| m["content"] == "a").unwrap();
|
||||
assert_eq!(first["reasoning"], "because");
|
||||
// No placeholder for a plain assistant turn — only tool-calling ones need it.
|
||||
let second = both.iter().find(|m| m["content"] == "a2").unwrap();
|
||||
assert!(second.get("reasoning_content").is_none());
|
||||
}
|
||||
|
||||
struct Digest;
|
||||
|
||||
#[async_trait]
|
||||
impl ToolResultDigest for Digest {
|
||||
async fn condense(&self, name: &str, _args: &Value, result: &str) -> Option<String> {
|
||||
Some(format!("[{name}: {} chars]", result.len()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn over_long_results_are_condensed_only_for_previous_turns() {
|
||||
let (store, frame) = store_and_frame("p7").await;
|
||||
|
||||
// Turn 1 (previous), then turn 2 (current), both with a long result.
|
||||
for (user, id) in [("first", "c1"), ("second", "c2")] {
|
||||
store.append(frame, NewMessage::user(user)).await.unwrap();
|
||||
let msg = store.append(frame, NewMessage::assistant("run", None)).await.unwrap();
|
||||
let call = store
|
||||
.append_call(msg, NewCall::new("read_file", json!({})).with_provider_id(id))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.resolve_call(call, &CallOutcome::Completed(ToolOutput::Text("x".repeat(100))))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let cfg = Projection {
|
||||
max_tool_result: Some(ResultLimit { max_chars: 10, previous_turns_only: true }),
|
||||
..Projection::default()
|
||||
};
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(cfg.clone())
|
||||
.with_digest(Arc::new(Digest))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let results: Vec<&str> = msgs
|
||||
.iter()
|
||||
.filter(|m| m["role"] == "tool")
|
||||
.map(|m| m["content"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(results[0], "[read_file: 100 chars]", "a previous turn is condensed");
|
||||
assert_eq!(results[1].len(), 100, "the current turn keeps its full output");
|
||||
|
||||
// Without a digest the crate truncates on a char boundary.
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(cfg)
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = msgs.iter().find(|m| m["role"] == "tool").unwrap();
|
||||
assert_eq!(first["content"], "xxxxxxxxxx… [truncated]");
|
||||
}
|
||||
|
||||
// ── dynamic tool loading ─────────────────────────────────────────────────────
|
||||
|
||||
/// An assistant turn with two calls, the activation being the SECOND one.
|
||||
async fn seed_two_calls(store: &Arc<dyn HistoryStore>, frame: FrameId) -> MessageId {
|
||||
store.append(frame, NewMessage::user("use gmail")).await.unwrap();
|
||||
let anchor = store.append(frame, NewMessage::assistant("activating", None)).await.unwrap();
|
||||
let other = store
|
||||
.append_call(anchor, NewCall::new("read_file", json!({})).with_provider_id("c1"))
|
||||
.await
|
||||
.unwrap();
|
||||
store.resolve_call(other, &CallOutcome::Completed(ToolOutput::Text("file".into()))).await.unwrap();
|
||||
let act = store
|
||||
.append_call(anchor, NewCall::new("activate_tools", json!({"groups":["gmail"]})).with_provider_id("c2"))
|
||||
.await
|
||||
.unwrap();
|
||||
store.resolve_call(act, &CallOutcome::Completed(ToolOutput::Text("activated".into()))).await.unwrap();
|
||||
anchor
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deferred_reference_marks_the_activation_result_not_the_first_one() {
|
||||
let (store, frame) = store_and_frame("p8").await;
|
||||
let anchor = seed_two_calls(&store, frame).await;
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(strict())
|
||||
.with_activation(Arc::new(Stub(vec![Activation {
|
||||
anchor,
|
||||
defs: vec![tool_def("mcp__gmail__send")],
|
||||
}])))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
|
||||
tool_rendering: ToolRendering::DeferredToolReference,
|
||||
..ModelInfo::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let tools: Vec<&Value> = msgs.iter().filter(|m| m["role"] == "tool").collect();
|
||||
assert!(tools[0].get("_tool_references").is_none(), "the read_file result is not the anchor");
|
||||
assert_eq!(tools[1]["_tool_references"], json!(["mcp__gmail__send"]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn system_tool_block_is_appended_after_the_result_group() {
|
||||
let (store, frame) = store_and_frame("p9").await;
|
||||
let anchor = seed_two_calls(&store, frame).await;
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(strict())
|
||||
.with_activation(Arc::new(Stub(vec![Activation {
|
||||
anchor,
|
||||
defs: vec![tool_def("mcp__gmail__send")],
|
||||
}])))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
|
||||
tool_rendering: ToolRendering::SystemToolBlock,
|
||||
..ModelInfo::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let idx = msgs
|
||||
.iter()
|
||||
.position(|m| m["role"] == "system" && m.get("tools").is_some())
|
||||
.expect("no system+tools block");
|
||||
assert_eq!(msgs[idx]["tools"][0]["function"]["name"], "mcp__gmail__send");
|
||||
assert!(msgs[idx].get("content").is_none(), "the block carries tools, not content");
|
||||
assert_eq!(msgs[idx - 1]["role"], "tool", "it comes right after the group");
|
||||
assert!(!msgs.iter().any(|m| m.get("_tool_references").is_some()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_mode_injects_nothing_at_all() {
|
||||
let (store, frame) = store_and_frame("p10").await;
|
||||
let anchor = seed_two_calls(&store, frame).await;
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_projection(strict())
|
||||
.with_activation(Arc::new(Stub(vec![Activation {
|
||||
anchor,
|
||||
defs: vec![tool_def("mcp__gmail__send")],
|
||||
}])))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!msgs.iter().any(|m| m.get("tools").is_some()));
|
||||
assert!(!msgs.iter().any(|m| m.get("_tool_references").is_some()));
|
||||
}
|
||||
|
||||
// ── media ────────────────────────────────────────────────────────────────────
|
||||
|
||||
struct Png(&'static str);
|
||||
|
||||
#[async_trait]
|
||||
impl MediaBlob for Png {
|
||||
fn name(&self) -> &str { self.0 }
|
||||
async fn size(&self) -> Option<u64> { Some(72) }
|
||||
async fn head(&self) -> Option<Vec<u8>> { Some(b"\x89PNG\r\n\x1a\n........".to_vec()) }
|
||||
async fn read_all(&self) -> Option<Vec<u8>> {
|
||||
let mut v = b"\x89PNG\r\n\x1a\n".to_vec();
|
||||
v.extend_from_slice(&[0xAA; 64]);
|
||||
Some(v)
|
||||
}
|
||||
}
|
||||
|
||||
/// Every user message has one image; every tool call produces one.
|
||||
struct Media;
|
||||
|
||||
#[async_trait]
|
||||
impl MediaSource for Media {
|
||||
async fn message_media(&self, _msg: &StoredMessage) -> Vec<Arc<dyn MediaBlob>> {
|
||||
vec![Arc::new(Png("shot.png"))]
|
||||
}
|
||||
async fn call_media(&self, _calls: &[StoredCall]) -> Vec<Arc<dyn MediaBlob>> {
|
||||
vec![Arc::new(Png("tool.png"))]
|
||||
}
|
||||
fn skipped_text(&self, _msg: &StoredMessage, skipped: &[usize]) -> Option<String> {
|
||||
(!skipped.is_empty()).then(|| format!("\n[files: {}]", skipped.len()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn media_is_inlined_for_the_current_turn_and_textual_before_it() {
|
||||
let (store, frame) = store_and_frame("p11").await;
|
||||
store.append(frame, NewMessage::user("old picture")).await.unwrap();
|
||||
store.append(frame, NewMessage::assistant("seen", None)).await.unwrap();
|
||||
store.append(frame, NewMessage::user("new picture")).await.unwrap();
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
|
||||
capabilities: vec!["vision".into()],
|
||||
..ModelInfo::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The previous turn keeps the textual note, no parts.
|
||||
assert_eq!(msgs[1], json!({ "role": "user", "content": "old picture\n[files: 1]" }));
|
||||
// The current turn inlines the bytes.
|
||||
let current = msgs.last().unwrap();
|
||||
assert_eq!(current["content"][0], json!({ "type": "text", "text": "new picture" }));
|
||||
assert!(
|
||||
current["content"][1]["image_url"]["url"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("data:image/png;base64,")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_model_without_vision_never_receives_bytes() {
|
||||
let (store, frame) = store_and_frame("p12").await;
|
||||
store.append(frame, NewMessage::user("picture")).await.unwrap();
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(msgs[1], json!({ "role": "user", "content": "picture\n[files: 1]" }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_produced_media_rides_a_synthetic_user_message_after_the_group() {
|
||||
let (store, frame) = store_and_frame("p13").await;
|
||||
store.append(frame, NewMessage::user("read the image")).await.unwrap();
|
||||
let msg = store.append(frame, NewMessage::assistant("reading", None)).await.unwrap();
|
||||
let call = store
|
||||
.append_call(msg, NewCall::new("read_file", json!({"path":"a.png"})).with_provider_id("c1"))
|
||||
.await
|
||||
.unwrap();
|
||||
store.resolve_call(call, &CallOutcome::Completed(ToolOutput::Text("image".into()))).await.unwrap();
|
||||
|
||||
let msgs = LinearAssembler::new()
|
||||
.with_media(Arc::new(Media))
|
||||
.build(&store, &input(frame, SystemContext::base("B"), ModelInfo {
|
||||
capabilities: vec!["vision".into()],
|
||||
..ModelInfo::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let last = msgs.last().unwrap();
|
||||
assert_eq!(last["role"], "user");
|
||||
assert_eq!(last["content"][0]["type"], "image_url");
|
||||
assert_eq!(msgs[msgs.len() - 2]["role"], "tool", "it follows the result group");
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
//! Recovery suite (blueprint §8/§13): the post-crash store is built **by hand**
|
||||
//! on `InMemoryStore` — a call left `Running`, a child frame nobody closed, two
|
||||
//! siblings of an interrupted batch — and recovery is asked to make it
|
||||
//! well-formed again and continue.
|
||||
//!
|
||||
//! No DB, no network: the states a real crash produces are exactly the states a
|
||||
//! test can write, because every transition is a store write.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use agent_loop::context::StaticSystemContext;
|
||||
use agent_loop::delegate::{AgentCatalog, AgentKind, AgentProfile, AgentSummary, ToolSelection};
|
||||
use agent_loop::ids::{ConversationId, FrameId, ToolCallId};
|
||||
use agent_loop::manager::{LoopManager, TurnMeta, TurnParams};
|
||||
use agent_loop::model::{ModelHint, StaticModels};
|
||||
use agent_loop::prelude::async_trait;
|
||||
use agent_loop::recovery::{HumanDecision, PendingPolicy, RecoveryPolicy, RunningPolicy};
|
||||
use agent_loop::store::{
|
||||
CallState, FrameSpec, HistoryStore, NewCall, NewMessage, StoredCall,
|
||||
};
|
||||
use agent_loop::store_memory::InMemoryStore;
|
||||
use agent_loop::testing::{self, FakeModel, Step};
|
||||
use agent_loop::tool::{
|
||||
RestartHint, Tool, ToolCtx, ToolFailure, ToolOutput, ToolRegistry, ToolSet,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
// ── tools ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Idempotent: safe to re-run after a crash. Counts its executions.
|
||||
struct Counter {
|
||||
runs: Arc<Mutex<usize>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for Counter {
|
||||
fn name(&self) -> &str { "counter" }
|
||||
fn definition(&self) -> Value {
|
||||
json!({"type":"function","function":{"name":"counter","parameters":{"type":"object"}}})
|
||||
}
|
||||
async fn call(&self, _args: Value, _ctx: &ToolCtx) -> Result<ToolOutput, ToolFailure> {
|
||||
let mut runs = self.runs.lock().unwrap();
|
||||
*runs += 1;
|
||||
Ok(ToolOutput::Text(format!("run {runs}")))
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-idempotent (a shell command already had its effect): must NOT be re-run.
|
||||
struct SideEffect {
|
||||
runs: Arc<Mutex<usize>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for SideEffect {
|
||||
fn name(&self) -> &str { "shell" }
|
||||
fn definition(&self) -> Value {
|
||||
json!({"type":"function","function":{"name":"shell","parameters":{"type":"object"}}})
|
||||
}
|
||||
fn restart_hint(&self) -> RestartHint { RestartHint::MarkInterrupted }
|
||||
async fn call(&self, _args: Value, _ctx: &ToolCtx) -> Result<ToolOutput, ToolFailure> {
|
||||
*self.runs.lock().unwrap() += 1;
|
||||
Ok(ToolOutput::Text("ran".into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Stands in for the delegate: recovery never calls it (a spawned frame is the
|
||||
/// cascade's business), so running it at all is a bug.
|
||||
struct NeverCalled;
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for NeverCalled {
|
||||
fn name(&self) -> &str { "delegate" }
|
||||
fn definition(&self) -> Value {
|
||||
json!({"type":"function","function":{"name":"delegate","parameters":{"type":"object"}}})
|
||||
}
|
||||
async fn call(&self, _args: Value, _ctx: &ToolCtx) -> Result<ToolOutput, ToolFailure> {
|
||||
panic!("recovery re-ran a sub-agent dispatch instead of cascading its frame");
|
||||
}
|
||||
}
|
||||
|
||||
// ── catalog ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Every child agent runs on the `child` model with its own prompt — so a test
|
||||
/// can prove a resumed sub-agent came back as ITSELF (B3), not as the root.
|
||||
struct Catalog;
|
||||
|
||||
#[async_trait]
|
||||
impl AgentCatalog for Catalog {
|
||||
async fn get(
|
||||
&self,
|
||||
id: &str,
|
||||
_child_frame: FrameId,
|
||||
_ctx: &ToolCtx,
|
||||
) -> agent_loop::Result<AgentProfile> {
|
||||
Ok(AgentProfile {
|
||||
id: id.into(),
|
||||
kind: AgentKind::Task,
|
||||
context: Arc::new(StaticSystemContext::new(format!("You are {id}."))),
|
||||
tools: ToolSelection::inherit(),
|
||||
toolset: None,
|
||||
model: Some(ModelHint::name("child")),
|
||||
selector: None,
|
||||
assembler: None,
|
||||
})
|
||||
}
|
||||
async fn list(&self, _kind: AgentKind) -> Vec<AgentSummary> { Vec::new() }
|
||||
}
|
||||
|
||||
// ── harness ──────────────────────────────────────────────────────────────────
|
||||
|
||||
struct H {
|
||||
manager: Arc<LoopManager>,
|
||||
store: Arc<InMemoryStore>,
|
||||
tools: Arc<dyn ToolSet>,
|
||||
conv: ConversationId,
|
||||
root: FrameId,
|
||||
counter: Arc<Mutex<usize>>,
|
||||
shell: Arc<Mutex<usize>>,
|
||||
/// The child's script — a test asserting "the model was NOT called" leaves
|
||||
/// it empty, and `FakeModel` panics if anything pops from it.
|
||||
child: Arc<FakeModel>,
|
||||
}
|
||||
|
||||
impl H {
|
||||
async fn new(root_script: Vec<Step>, child_script: Vec<Step>) -> Self {
|
||||
let store = Arc::new(InMemoryStore::new());
|
||||
let root_model = Arc::new(FakeModel::new("root", root_script));
|
||||
let child = Arc::new(FakeModel::new("child", child_script));
|
||||
let manager = Arc::new(
|
||||
LoopManager::builder()
|
||||
.models(Arc::new(StaticModels::new(vec![
|
||||
testing::handle(&root_model, "root"),
|
||||
testing::handle(&child, "child"),
|
||||
])))
|
||||
.store(store.clone())
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
let counter = Arc::new(Mutex::new(0));
|
||||
let shell = Arc::new(Mutex::new(0));
|
||||
let tools: Arc<dyn ToolSet> = ToolRegistry::new()
|
||||
.with(Counter { runs: counter.clone() })
|
||||
.with(SideEffect { runs: shell.clone() })
|
||||
.with(NeverCalled)
|
||||
.into_toolset();
|
||||
|
||||
let conv = ConversationId::new("rec");
|
||||
let root = store
|
||||
.open_frame(&conv, None, FrameSpec::root("assistant"))
|
||||
.await
|
||||
.unwrap();
|
||||
Self { manager, store, tools, conv, root, counter, shell, child }
|
||||
}
|
||||
|
||||
fn params(&self) -> TurnParams {
|
||||
TurnParams {
|
||||
frame: self.root,
|
||||
agent: "assistant".into(),
|
||||
system: Arc::new(StaticSystemContext::new("You are the assistant.")),
|
||||
tools: self.tools.clone(),
|
||||
model_hint: ModelHint::default(),
|
||||
selector: None,
|
||||
live_input: None,
|
||||
extensions: Default::default(),
|
||||
meta: TurnMeta::default(),
|
||||
assembler: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// An assistant message with one call left in flight — what a crash leaves.
|
||||
async fn interrupted_call(&self, frame: FrameId, name: &str) -> ToolCallId {
|
||||
self.store.append(frame, NewMessage::user("do it")).await.unwrap();
|
||||
let msg = self
|
||||
.store
|
||||
.append(frame, NewMessage::assistant("calling", None))
|
||||
.await
|
||||
.unwrap();
|
||||
self.store.append_call(msg, NewCall::new(name, json!({}))).await.unwrap()
|
||||
}
|
||||
|
||||
/// A child frame spawned by `call`, with its prompt already appended.
|
||||
async fn child_frame(&self, agent: &str, call: ToolCallId) -> FrameId {
|
||||
let frame = self
|
||||
.store
|
||||
.open_frame(&self.conv, Some(self.root), FrameSpec {
|
||||
agent: agent.into(),
|
||||
prompt: Some("go find out".into()),
|
||||
depth: 1,
|
||||
parent_call: Some(call),
|
||||
meta: Value::Null,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
self.store.append(frame, NewMessage::agent("go find out")).await.unwrap();
|
||||
frame
|
||||
}
|
||||
|
||||
async fn call(&self, id: ToolCallId) -> StoredCall {
|
||||
self.store.get_call(id).await.unwrap().unwrap()
|
||||
}
|
||||
|
||||
async fn recover_with(&self, policy: RecoveryPolicy) -> agent_loop::recovery::RecoveryReport {
|
||||
let recovery = self.manager.recovery(Arc::new(Catalog), policy);
|
||||
tokio::time::timeout(Duration::from_secs(5), recovery.run(&self.conv, &self.params()))
|
||||
.await
|
||||
.expect("recovery hung")
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn recover(&self) -> agent_loop::recovery::RecoveryReport {
|
||||
self.recover_with(RecoveryPolicy::default()).await
|
||||
}
|
||||
}
|
||||
|
||||
// ── interrupted calls ────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_interrupted_idempotent_call_is_re_executed_then_the_turn_continues() {
|
||||
let h = H::new(vec![Step::message("all done")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "counter").await;
|
||||
|
||||
let report = h.recover().await;
|
||||
|
||||
assert_eq!(*h.counter.lock().unwrap(), 1, "the call must run exactly once");
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Done);
|
||||
assert_eq!(call.result.as_deref(), Some("run 1"));
|
||||
assert_eq!(report.calls_reexecuted, 1);
|
||||
assert_eq!(report.frames_resumed, 1, "the frame then ran a normal round");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_interrupted_call_with_side_effects_is_failed_not_re_run() {
|
||||
// D7: `shell` declares MarkInterrupted, so re-running it could repeat an
|
||||
// effect that already happened.
|
||||
let h = H::new(vec![Step::message("I stopped mid-command")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "shell").await;
|
||||
|
||||
let report = h.recover().await;
|
||||
|
||||
assert_eq!(*h.shell.lock().unwrap(), 0, "a non-idempotent tool must NOT be re-run");
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Failed);
|
||||
assert!(call.result.as_deref().unwrap().contains("interrupted"), "{:?}", call.result);
|
||||
assert_eq!(report.calls_failed, 1);
|
||||
assert_eq!(report.calls_reexecuted, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_policy_can_refuse_to_re_run_anything() {
|
||||
let h = H::new(vec![Step::message("continuing")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "counter").await;
|
||||
|
||||
h.recover_with(RecoveryPolicy {
|
||||
on_running: RunningPolicy::MarkInterrupted,
|
||||
..RecoveryPolicy::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(*h.counter.lock().unwrap(), 0, "the policy overrides the tool's hint");
|
||||
assert_eq!(h.call(call).await.state, CallState::Failed);
|
||||
}
|
||||
|
||||
// ── awaiting human ───────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_call_awaiting_a_human_is_asked_again() {
|
||||
let h = H::new(vec![Step::message("approved and done")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "counter").await;
|
||||
h.store.set_call_state(call, CallState::AwaitingHuman).await.unwrap();
|
||||
|
||||
let report = h.recover().await;
|
||||
|
||||
// ReAsk re-runs it through the gate — here an allowing one, so it executes.
|
||||
assert_eq!(*h.counter.lock().unwrap(), 1);
|
||||
assert_eq!(h.call(call).await.state, CallState::Done);
|
||||
assert_eq!(report.calls_reexecuted, 1);
|
||||
assert!(!report.left_pending);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn leave_pending_stops_and_touches_nothing() {
|
||||
// No model step scripted: running the loop would panic, which is the point —
|
||||
// a frame with an unanswered call must not be driven.
|
||||
let h = H::new(vec![], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "counter").await;
|
||||
h.store.set_call_state(call, CallState::AwaitingHuman).await.unwrap();
|
||||
|
||||
let report = h.recover_with(RecoveryPolicy {
|
||||
on_awaiting_human: PendingPolicy::LeavePending,
|
||||
..RecoveryPolicy::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(report.left_pending);
|
||||
assert_eq!(report.frames_resumed, 0);
|
||||
assert_eq!(*h.counter.lock().unwrap(), 0);
|
||||
assert_eq!(h.call(call).await.state, CallState::AwaitingHuman, "still the human's to answer");
|
||||
}
|
||||
|
||||
// ── the cascade ──────────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_interrupted_sub_agent_finishes_as_itself_then_the_parent_continues() {
|
||||
let h = H::new(
|
||||
vec![Step::message("the root's final answer")],
|
||||
vec![Step::message("the child's answer")],
|
||||
)
|
||||
.await;
|
||||
let call = h.interrupted_call(h.root, "delegate").await;
|
||||
let child = h.child_frame("researcher", call).await;
|
||||
|
||||
let report = h.recover().await;
|
||||
|
||||
// The child ran under ITS agent's prompt and model (B3), not the root's.
|
||||
let seen = h.child.requests();
|
||||
assert_eq!(seen.len(), 1, "the child model ran exactly once");
|
||||
assert!(
|
||||
serde_json::to_string(&seen[0].messages).unwrap().contains("You are researcher."),
|
||||
"the resumed frame must run its own agent's context: {:?}",
|
||||
seen[0].messages
|
||||
);
|
||||
|
||||
// Its answer became the parent call's result, and the child frame is closed.
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Done);
|
||||
assert_eq!(call.result.as_deref(), Some("the child's answer"));
|
||||
assert!(!h.store.get_frame(child).await.unwrap().unwrap().active);
|
||||
assert_eq!(report.frames_resumed, 2, "child then root");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_child_that_finished_but_never_propagated_is_not_re_run() {
|
||||
// The wedge: the turn died in the instant between the child's last message
|
||||
// and its result reaching the parent. Re-running the model would ask it to
|
||||
// answer a question it already answered — the empty child script asserts
|
||||
// that never happens.
|
||||
let h = H::new(vec![Step::message("root wraps up")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "delegate").await;
|
||||
let child = h.child_frame("researcher", call).await;
|
||||
h.store
|
||||
.append(child, NewMessage::assistant("already done", None))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let report = h.recover().await;
|
||||
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Done);
|
||||
assert_eq!(call.result.as_deref(), Some("already done"));
|
||||
assert_eq!(h.child.requests().len(), 0, "the child's LLM must not be called again");
|
||||
assert_eq!(report.frames_resumed, 1, "only the parent ran");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_interrupted_parallel_batch_is_reaped_and_the_parent_resumes() {
|
||||
let h = H::new(vec![Step::message("carrying on without them")], vec![]).await;
|
||||
|
||||
// Two delegate calls in one round, two live children: impossible for a
|
||||
// linear stack, so it can only be a batch caught mid-flight.
|
||||
h.store.append(h.root, NewMessage::user("do both")).await.unwrap();
|
||||
let msg = h.store.append(h.root, NewMessage::assistant("", None)).await.unwrap();
|
||||
let c1 = h.store.append_call(msg, NewCall::new("delegate", json!({}))).await.unwrap();
|
||||
let c2 = h.store.append_call(msg, NewCall::new("delegate", json!({}))).await.unwrap();
|
||||
let f1 = h.child_frame("a1", c1).await;
|
||||
let f2 = h.child_frame("a2", c2).await;
|
||||
|
||||
let report = h.recover().await;
|
||||
|
||||
assert_eq!(report.batches_reaped, 1);
|
||||
for (call, frame) in [(c1, f1), (c2, f2)] {
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Failed);
|
||||
assert!(call.result.as_deref().unwrap().contains("parallel batch"), "{:?}", call.result);
|
||||
assert!(!h.store.get_frame(frame).await.unwrap().unwrap().active);
|
||||
}
|
||||
assert_eq!(h.child.requests().len(), 0, "a reaped batch is not re-run");
|
||||
assert_eq!(report.frames_resumed, 1, "the root continues with the failures in view");
|
||||
}
|
||||
|
||||
// ── resolve_pending ──────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn approving_after_a_restart_runs_the_call_and_continues() {
|
||||
let h = H::new(vec![Step::message("done, as approved")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "counter").await;
|
||||
h.store.set_call_state(call, CallState::AwaitingHuman).await.unwrap();
|
||||
|
||||
h.manager
|
||||
.resolve_pending(call, HumanDecision::Approved, Arc::new(Catalog), &h.params())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(*h.counter.lock().unwrap(), 1);
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Done);
|
||||
assert_eq!(call.result.as_deref(), Some("run 1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejecting_after_a_restart_records_the_refusal_and_continues() {
|
||||
let h = H::new(vec![Step::message("understood, I won't")], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "shell").await;
|
||||
h.store.set_call_state(call, CallState::AwaitingHuman).await.unwrap();
|
||||
|
||||
h.manager
|
||||
.resolve_pending(
|
||||
call,
|
||||
HumanDecision::Rejected { reason: "no thanks".into() },
|
||||
Arc::new(Catalog),
|
||||
&h.params(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(*h.shell.lock().unwrap(), 0);
|
||||
let call = h.call(call).await;
|
||||
assert_eq!(call.state, CallState::Rejected);
|
||||
assert_eq!(call.result.as_deref(), Some("no thanks"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolving_an_already_terminal_call_is_a_no_op() {
|
||||
let h = H::new(vec![], vec![]).await;
|
||||
let call = h.interrupted_call(h.root, "counter").await;
|
||||
h.store
|
||||
.resolve_call(call, &agent_loop::store::CallOutcome::Cancelled)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let report = h
|
||||
.manager
|
||||
.resolve_pending(call, HumanDecision::Approved, Arc::new(Catalog), &h.params())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Cancelled is terminal and never re-executed (blueprint §8.2).
|
||||
assert_eq!(*h.counter.lock().unwrap(), 0);
|
||||
assert_eq!(h.call(call).await.state, CallState::Cancelled);
|
||||
assert_eq!(report.frames_resumed, 0);
|
||||
}
|
||||
Reference in New Issue
Block a user