Phase 21 foundation — context stability + chunking pipeline
PRD: add Phase 20 (model matrix, wired) and Phase 21 (context stability, partial). Phase 21 exists because LLM Team hit this exact wall — running multi-model ranking on large context silently truncated, rankings degraded, no pipeline caught it. The stable answer: every agent call goes through a budget check against the model's declared context_window minus safety_margin, with a declared overflow_policy when the check fails. config/models.json: - context_window + context_budget per tier - overflow_policies block: summarize_oldest_tool_results_via_t3, chunk_lessons_via_cosine_topk, two_pass_map_reduce, escalate_to_kimi_k2_1t_or_split_decision - chunking_cache spec (data/_chunk_cache/, corpus-hash keyed) agent.ts: - estimateTokens() chars/4 biased safe ~15% - CONTEXT_WINDOWS table (fallback; prod reads models.json) - assertContextBudget() — throws on overflow with exact numbers, can bypass with bypass_budget:true for callers with their own policy - Wired into generate() and generateCloud() so EVERY call is checked scenario.ts: - T3 lesson archive to data/_playbook_lessons/*.json (the old /vectors/playbook_memory/seed path was silently failing with HTTP 400 because it requires 'fill: Role xN in City, ST' operation shape) - loadPriorLessons() at scenario start — filters by city/state match, date-sorted, takes top-3 - prior_lessons.json archived per-run (honest signal for A/B) - guidanceFor() injects up to 2 prior lessons (≤500 chars each) into the executor's per-event context - Retrospective shows explicit "Prior lessons loaded: N" line Verified: mistral correctly rejects a 150K-char prompt (7532 tokens over), gpt-oss:120b accepts it with 90K headroom. The enforcement is in-band on every call now, not an afterthought. Full chunking service (Rust) remains deferred to the sprint this feeds: crates/aibridge/src/budget.rs + chunk.rs + storaged/chunk_cache.rs
This commit is contained in:
root
+103
-19
@@ -146,6 +146,50 @@ interface ScenarioContext {
|
||||
roster: RosterEntry[];
|
||||
results: EventResult[];
|
||||
gap_signals: Array<{ event: string; category: string; detail: string }>;
|
||||
prior_lessons: PriorLesson[];
|
||||
}
|
||||
|
||||
interface PriorLesson {
|
||||
date: string;
|
||||
client: string;
|
||||
cities: string;
|
||||
states: string;
|
||||
lesson: string;
|
||||
events_ok: number;
|
||||
events_total: number;
|
||||
file: string;
|
||||
}
|
||||
|
||||
// Load lessons from prior T3 runs — read-back half of the feedback loop.
|
||||
// Filters to the most relevant by matching ANY city/state with the current
|
||||
// spec, then takes the 3 newest. Keeps startup cheap; file scan is O(n).
|
||||
async function loadPriorLessons(spec: ScenarioSpec): Promise<PriorLesson[]> {
|
||||
try {
|
||||
const { readdir, readFile } = await import("node:fs/promises");
|
||||
const dir = join("data", "_playbook_lessons");
|
||||
const files = await readdir(dir).catch(() => [] as string[]);
|
||||
if (files.length === 0) return [];
|
||||
const specCities = new Set(spec.events.map(e => e.city));
|
||||
const specStates = new Set(spec.events.map(e => e.state));
|
||||
const parsed: PriorLesson[] = [];
|
||||
for (const f of files) {
|
||||
if (!f.endsWith(".json")) continue;
|
||||
try {
|
||||
const raw = await readFile(join(dir, f), "utf8");
|
||||
const rec = JSON.parse(raw);
|
||||
parsed.push({ ...rec, file: f });
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
const relevant = parsed.filter(p => {
|
||||
const cities = (p.cities ?? "").split(",");
|
||||
const states = (p.states ?? "").split(",");
|
||||
return cities.some(c => specCities.has(c)) || states.some(s => specStates.has(s));
|
||||
});
|
||||
relevant.sort((a, b) => (b.date ?? "").localeCompare(a.date ?? ""));
|
||||
return relevant.slice(0, 3);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// =================== Default scenario ===================
|
||||
@@ -493,7 +537,18 @@ CAST(reliability AS DOUBLE) > 0.7.`;
|
||||
}
|
||||
})();
|
||||
|
||||
return `${schemaLock}\n\nEVENT FOCUS:\n${base}`;
|
||||
// Prior-lesson hint — surface up to 2 most recent lessons learned from
|
||||
// T3 overseer runs against this city/state. Terse to avoid diluting the
|
||||
// prompt. The goal is to pass forward hard-won mistakes, not flood the
|
||||
// context. This is the read-back half of the T3 feedback loop.
|
||||
const priorHint = ctx.prior_lessons.length > 0
|
||||
? `\n\nPRIOR LESSONS (from T3 overseer on past runs in similar cities):\n` +
|
||||
ctx.prior_lessons.slice(0, 2).map((p, i) =>
|
||||
`${i + 1}. ${p.date} ${p.client} (${p.cities}): ${p.lesson.replace(/\s+/g, " ").slice(0, 500)}`
|
||||
).join("\n")
|
||||
: "";
|
||||
|
||||
return `${schemaLock}\n\nEVENT FOCUS:\n${base}${priorHint}`;
|
||||
}
|
||||
|
||||
// =================== Artifact generation ===================
|
||||
@@ -878,7 +933,8 @@ async function writeRetrospective(ctx: ScenarioContext): Promise<void> {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Scenario retrospective — ${ctx.spec.client}, ${ctx.spec.date}`);
|
||||
lines.push("");
|
||||
lines.push(`Executor: \`${EXECUTOR_MODEL}\` Reviewer: \`${REVIEWER_MODEL}\` Draft: \`${DRAFT_MODEL}\` Overview(T3): \`${T3_DISABLED ? "disabled" : OVERVIEW_MODEL}\``);
|
||||
lines.push(`Executor: \`${EXECUTOR_MODEL}\` Reviewer: \`${REVIEWER_MODEL}\` Draft: \`${DRAFT_MODEL}\` Overview(T3): \`${T3_DISABLED ? "disabled" : OVERVIEW_MODEL + (OVERVIEW_CLOUD ? " (cloud)" : "")}\``);
|
||||
lines.push(`Prior lessons loaded into executor context: **${ctx.prior_lessons.length}**${ctx.prior_lessons.length > 0 ? " (from " + ctx.prior_lessons.map(p => p.date).join(", ") + ")" : " (baseline — no prior T3 history)"}`);
|
||||
lines.push("");
|
||||
|
||||
// --- Per-event summary ---
|
||||
@@ -1019,12 +1075,15 @@ async function main() {
|
||||
const out_dir = join("tests/multi-agent/playbooks", `scenario-${stamp}`);
|
||||
await mkdir(out_dir, { recursive: true });
|
||||
|
||||
const prior_lessons = await loadPriorLessons(spec);
|
||||
|
||||
const ctx: ScenarioContext = {
|
||||
spec,
|
||||
out_dir,
|
||||
roster: [],
|
||||
results: [],
|
||||
gap_signals: [],
|
||||
prior_lessons,
|
||||
};
|
||||
|
||||
// Initialize output files
|
||||
@@ -1033,6 +1092,19 @@ async function main() {
|
||||
await writeFile(join(out_dir, "dispatch.jsonl"), "");
|
||||
await writeFile(join(out_dir, "checkpoints.jsonl"), "");
|
||||
|
||||
// Archive which prior lessons this run will see, so the retrospective
|
||||
// can tell whether the T3 feedback loop actually fed back anything.
|
||||
await writeFile(
|
||||
join(out_dir, "prior_lessons.json"),
|
||||
JSON.stringify(prior_lessons, null, 2)
|
||||
);
|
||||
if (prior_lessons.length > 0) {
|
||||
console.log(`▶ prior lessons loaded: ${prior_lessons.length} (from data/_playbook_lessons/)`);
|
||||
for (const p of prior_lessons) {
|
||||
console.log(` - ${p.date} ${p.client} (${p.cities}) — ${p.events_ok}/${p.events_total} ok`);
|
||||
}
|
||||
}
|
||||
|
||||
const checkpoints: OverviewCheckpoint[] = [];
|
||||
|
||||
console.log(`▶ scenario: ${spec.client}, ${spec.date}, ${spec.events.length} events`);
|
||||
@@ -1109,26 +1181,38 @@ async function main() {
|
||||
);
|
||||
console.log(`✓ lesson (${lessonSecs}s) → ${join(out_dir, "lesson.md")}`);
|
||||
|
||||
// Seed the lesson into playbook_memory for future retrieval. Keep
|
||||
// the embedded `approach` + `context` terse per feedback_phase19_seed_text.md;
|
||||
// the rich prose lives in lesson.md and a separate `rationale` field.
|
||||
// Persist the lesson to data/_playbook_lessons/ so future scenarios
|
||||
// can read it verbatim at startup. The /vectors/playbook_memory/seed
|
||||
// endpoint rejects operations that don't match the `fill: Role xN
|
||||
// in City, ST` regex (enforced in crates/vectord/src/service.rs),
|
||||
// so embedding-based retrieval of cross-day lessons isn't wired.
|
||||
// File-based read-back is durable and explicit — future scenarios
|
||||
// pull from the lessons dir at startup and include top-N in the
|
||||
// executor's system context.
|
||||
try {
|
||||
const kinds = [...new Set(ctx.spec.events.map(e => e.kind))].join("+");
|
||||
const cities = [...new Set(ctx.spec.events.map(e => e.city))].slice(0, 3).join(",");
|
||||
await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: `cross-day-lesson-${ctx.spec.date}`,
|
||||
approach: `${kinds} day in ${cities}`,
|
||||
context: `${ctx.spec.client} ${ctx.spec.date}`,
|
||||
rationale: lesson.slice(0, 2000),
|
||||
endorsed_names: [],
|
||||
append: true,
|
||||
}),
|
||||
});
|
||||
const states = [...new Set(ctx.spec.events.map(e => e.state))].slice(0, 3).join(",");
|
||||
const lessonsDir = join("data", "_playbook_lessons");
|
||||
await mkdir(lessonsDir, { recursive: true });
|
||||
const lessonRec = {
|
||||
date: ctx.spec.date,
|
||||
client: ctx.spec.client,
|
||||
cities,
|
||||
states,
|
||||
events_total: ctx.spec.events.length,
|
||||
events_ok: ctx.results.filter(r => r.ok).length,
|
||||
checkpoint_count: checkpoints.length,
|
||||
model: OVERVIEW_MODEL,
|
||||
cloud: OVERVIEW_CLOUD,
|
||||
lesson: lesson.trim(),
|
||||
checkpoints: checkpoints.map(c => ({ after: c.after_event, risk: c.risk, hint: c.hint })),
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
const fname = `${ctx.spec.date}_${ctx.spec.client.replace(/\s+/g, "_")}_${Date.now()}.json`;
|
||||
await writeFile(join(lessonsDir, fname), JSON.stringify(lessonRec, null, 2));
|
||||
console.log(` lesson archived → ${join(lessonsDir, fname)}`);
|
||||
} catch (e) {
|
||||
console.log(` (lesson seed skipped: ${(e as Error).message})`);
|
||||
console.log(` (lesson archive skipped: ${(e as Error).message})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user