Phase 21 foundation — context stability + chunking pipeline
PRD: add Phase 20 (model matrix, wired) and Phase 21 (context stability, partial). Phase 21 exists because LLM Team hit this exact wall — running multi-model ranking on large context silently truncated, rankings degraded, no pipeline caught it. The stable answer: every agent call goes through a budget check against the model's declared context_window minus safety_margin, with a declared overflow_policy when the check fails. config/models.json: - context_window + context_budget per tier - overflow_policies block: summarize_oldest_tool_results_via_t3, chunk_lessons_via_cosine_topk, two_pass_map_reduce, escalate_to_kimi_k2_1t_or_split_decision - chunking_cache spec (data/_chunk_cache/, corpus-hash keyed) agent.ts: - estimateTokens() chars/4 biased safe ~15% - CONTEXT_WINDOWS table (fallback; prod reads models.json) - assertContextBudget() — throws on overflow with exact numbers, can bypass with bypass_budget:true for callers with their own policy - Wired into generate() and generateCloud() so EVERY call is checked scenario.ts: - T3 lesson archive to data/_playbook_lessons/*.json (the old /vectors/playbook_memory/seed path was silently failing with HTTP 400 because it requires 'fill: Role xN in City, ST' operation shape) - loadPriorLessons() at scenario start — filters by city/state match, date-sorted, takes top-3 - prior_lessons.json archived per-run (honest signal for A/B) - guidanceFor() injects up to 2 prior lessons (≤500 chars each) into the executor's per-event context - Retrospective shows explicit "Prior lessons loaded: N" line Verified: mistral correctly rejects a 150K-char prompt (7532 tokens over), gpt-oss:120b accepts it with 90K headroom. The enforcement is in-band on every call now, not an afterthought. Full chunking service (Rust) remains deferred to the sprint this feeds: crates/aibridge/src/budget.rs + chunk.rs + storaged/chunk_cache.rs
This commit is contained in:
root
@@ -15,6 +15,54 @@ export const SIDECAR = "http://localhost:3200";
|
||||
export const OLLAMA_CLOUD_URL = process.env.OLLAMA_CLOUD_URL ?? "https://ollama.com";
|
||||
export const OLLAMA_CLOUD_KEY = process.env.OLLAMA_CLOUD_KEY ?? "";
|
||||
|
||||
// Rough token estimator — chars/4 biased safe by ~15%. Swap to a real
|
||||
// tokenizer (tiktoken or provider endpoint) once Phase 21 lands. Good
|
||||
// enough to stop the "context silently truncated" failure mode today.
|
||||
export function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
// Known context windows — matches crates/../config/models.json. Kept in
|
||||
// code as a fallback so the test harness doesn't crash if the config is
|
||||
// missing. Production path should read from models.json.
|
||||
export const CONTEXT_WINDOWS: Record<string, number> = {
|
||||
"mistral:latest": 32768,
|
||||
"qwen2.5:latest": 32768,
|
||||
"qwen3:latest": 40960,
|
||||
"gpt-oss:20b": 131072,
|
||||
"gpt-oss:120b": 131072,
|
||||
"qwen3.5:397b": 131072,
|
||||
"kimi-k2-thinking": 200000,
|
||||
"kimi-k2:1t": 1048576,
|
||||
"deepseek-v3.1:671b": 131072,
|
||||
"glm-4.7": 131072,
|
||||
};
|
||||
const DEFAULT_CONTEXT_WINDOW = 32768;
|
||||
const DEFAULT_SAFETY_MARGIN = 2000;
|
||||
|
||||
// Fail LOUDLY if a prompt would blow the model's context. The whole
|
||||
// point of Phase 21 is to stop silent truncation — so we throw with the
|
||||
// numbers. Callers that expect to handle overflow should chunk BEFORE
|
||||
// calling; they can also set bypass: true to opt out (T5 gatekeeper
|
||||
// handles its own overflow policy).
|
||||
export function assertContextBudget(
|
||||
model: string,
|
||||
prompt: string,
|
||||
opts: { system?: string; max_tokens?: number; safety_margin?: number; bypass?: boolean } = {}
|
||||
): { estimated: number; window: number; remaining: number } {
|
||||
const window = CONTEXT_WINDOWS[model] ?? DEFAULT_CONTEXT_WINDOW;
|
||||
const safety = opts.safety_margin ?? DEFAULT_SAFETY_MARGIN;
|
||||
const estimated = estimateTokens(prompt) + estimateTokens(opts.system ?? "") + (opts.max_tokens ?? 800);
|
||||
const remaining = window - estimated - safety;
|
||||
if (remaining < 0 && !opts.bypass) {
|
||||
throw new Error(
|
||||
`context overflow: model=${model} est=${estimated}t window=${window}t safety=${safety}t over=${-remaining}t. ` +
|
||||
`Chunk the prompt (see config/models.json overflow_policies) or set bypass:true if you know the risk.`
|
||||
);
|
||||
}
|
||||
return { estimated, window, remaining };
|
||||
}
|
||||
|
||||
// --- Shared types ---
|
||||
|
||||
export type Role = "executor" | "reviewer";
|
||||
@@ -101,7 +149,13 @@ export async function generate(model: string, prompt: string, opts: {
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
system?: string;
|
||||
bypass_budget?: boolean;
|
||||
} = {}): Promise<string> {
|
||||
assertContextBudget(model, prompt, {
|
||||
system: opts.system,
|
||||
max_tokens: opts.max_tokens,
|
||||
bypass: opts.bypass_budget,
|
||||
});
|
||||
const body: Record<string, any> = {
|
||||
model,
|
||||
prompt,
|
||||
@@ -126,10 +180,16 @@ export async function generateCloud(model: string, prompt: string, opts: {
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
system?: string;
|
||||
bypass_budget?: boolean;
|
||||
} = {}): Promise<string> {
|
||||
if (!OLLAMA_CLOUD_KEY) {
|
||||
throw new Error("OLLAMA_CLOUD_KEY not set; cannot reach Ollama Cloud");
|
||||
}
|
||||
assertContextBudget(model, prompt, {
|
||||
system: opts.system,
|
||||
max_tokens: opts.max_tokens,
|
||||
bypass: opts.bypass_budget,
|
||||
});
|
||||
const body: Record<string, any> = {
|
||||
model,
|
||||
prompt,
|
||||
|
||||
+103
-19
@@ -146,6 +146,50 @@ interface ScenarioContext {
|
||||
roster: RosterEntry[];
|
||||
results: EventResult[];
|
||||
gap_signals: Array<{ event: string; category: string; detail: string }>;
|
||||
prior_lessons: PriorLesson[];
|
||||
}
|
||||
|
||||
interface PriorLesson {
|
||||
date: string;
|
||||
client: string;
|
||||
cities: string;
|
||||
states: string;
|
||||
lesson: string;
|
||||
events_ok: number;
|
||||
events_total: number;
|
||||
file: string;
|
||||
}
|
||||
|
||||
// Load lessons from prior T3 runs — read-back half of the feedback loop.
|
||||
// Filters to the most relevant by matching ANY city/state with the current
|
||||
// spec, then takes the 3 newest. Keeps startup cheap; file scan is O(n).
|
||||
async function loadPriorLessons(spec: ScenarioSpec): Promise<PriorLesson[]> {
|
||||
try {
|
||||
const { readdir, readFile } = await import("node:fs/promises");
|
||||
const dir = join("data", "_playbook_lessons");
|
||||
const files = await readdir(dir).catch(() => [] as string[]);
|
||||
if (files.length === 0) return [];
|
||||
const specCities = new Set(spec.events.map(e => e.city));
|
||||
const specStates = new Set(spec.events.map(e => e.state));
|
||||
const parsed: PriorLesson[] = [];
|
||||
for (const f of files) {
|
||||
if (!f.endsWith(".json")) continue;
|
||||
try {
|
||||
const raw = await readFile(join(dir, f), "utf8");
|
||||
const rec = JSON.parse(raw);
|
||||
parsed.push({ ...rec, file: f });
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
const relevant = parsed.filter(p => {
|
||||
const cities = (p.cities ?? "").split(",");
|
||||
const states = (p.states ?? "").split(",");
|
||||
return cities.some(c => specCities.has(c)) || states.some(s => specStates.has(s));
|
||||
});
|
||||
relevant.sort((a, b) => (b.date ?? "").localeCompare(a.date ?? ""));
|
||||
return relevant.slice(0, 3);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// =================== Default scenario ===================
|
||||
@@ -493,7 +537,18 @@ CAST(reliability AS DOUBLE) > 0.7.`;
|
||||
}
|
||||
})();
|
||||
|
||||
return `${schemaLock}\n\nEVENT FOCUS:\n${base}`;
|
||||
// Prior-lesson hint — surface up to 2 most recent lessons learned from
|
||||
// T3 overseer runs against this city/state. Terse to avoid diluting the
|
||||
// prompt. The goal is to pass forward hard-won mistakes, not flood the
|
||||
// context. This is the read-back half of the T3 feedback loop.
|
||||
const priorHint = ctx.prior_lessons.length > 0
|
||||
? `\n\nPRIOR LESSONS (from T3 overseer on past runs in similar cities):\n` +
|
||||
ctx.prior_lessons.slice(0, 2).map((p, i) =>
|
||||
`${i + 1}. ${p.date} ${p.client} (${p.cities}): ${p.lesson.replace(/\s+/g, " ").slice(0, 500)}`
|
||||
).join("\n")
|
||||
: "";
|
||||
|
||||
return `${schemaLock}\n\nEVENT FOCUS:\n${base}${priorHint}`;
|
||||
}
|
||||
|
||||
// =================== Artifact generation ===================
|
||||
@@ -878,7 +933,8 @@ async function writeRetrospective(ctx: ScenarioContext): Promise<void> {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Scenario retrospective — ${ctx.spec.client}, ${ctx.spec.date}`);
|
||||
lines.push("");
|
||||
lines.push(`Executor: \`${EXECUTOR_MODEL}\` Reviewer: \`${REVIEWER_MODEL}\` Draft: \`${DRAFT_MODEL}\` Overview(T3): \`${T3_DISABLED ? "disabled" : OVERVIEW_MODEL}\``);
|
||||
lines.push(`Executor: \`${EXECUTOR_MODEL}\` Reviewer: \`${REVIEWER_MODEL}\` Draft: \`${DRAFT_MODEL}\` Overview(T3): \`${T3_DISABLED ? "disabled" : OVERVIEW_MODEL + (OVERVIEW_CLOUD ? " (cloud)" : "")}\``);
|
||||
lines.push(`Prior lessons loaded into executor context: **${ctx.prior_lessons.length}**${ctx.prior_lessons.length > 0 ? " (from " + ctx.prior_lessons.map(p => p.date).join(", ") + ")" : " (baseline — no prior T3 history)"}`);
|
||||
lines.push("");
|
||||
|
||||
// --- Per-event summary ---
|
||||
@@ -1019,12 +1075,15 @@ async function main() {
|
||||
const out_dir = join("tests/multi-agent/playbooks", `scenario-${stamp}`);
|
||||
await mkdir(out_dir, { recursive: true });
|
||||
|
||||
const prior_lessons = await loadPriorLessons(spec);
|
||||
|
||||
const ctx: ScenarioContext = {
|
||||
spec,
|
||||
out_dir,
|
||||
roster: [],
|
||||
results: [],
|
||||
gap_signals: [],
|
||||
prior_lessons,
|
||||
};
|
||||
|
||||
// Initialize output files
|
||||
@@ -1033,6 +1092,19 @@ async function main() {
|
||||
await writeFile(join(out_dir, "dispatch.jsonl"), "");
|
||||
await writeFile(join(out_dir, "checkpoints.jsonl"), "");
|
||||
|
||||
// Archive which prior lessons this run will see, so the retrospective
|
||||
// can tell whether the T3 feedback loop actually fed back anything.
|
||||
await writeFile(
|
||||
join(out_dir, "prior_lessons.json"),
|
||||
JSON.stringify(prior_lessons, null, 2)
|
||||
);
|
||||
if (prior_lessons.length > 0) {
|
||||
console.log(`▶ prior lessons loaded: ${prior_lessons.length} (from data/_playbook_lessons/)`);
|
||||
for (const p of prior_lessons) {
|
||||
console.log(` - ${p.date} ${p.client} (${p.cities}) — ${p.events_ok}/${p.events_total} ok`);
|
||||
}
|
||||
}
|
||||
|
||||
const checkpoints: OverviewCheckpoint[] = [];
|
||||
|
||||
console.log(`▶ scenario: ${spec.client}, ${spec.date}, ${spec.events.length} events`);
|
||||
@@ -1109,26 +1181,38 @@ async function main() {
|
||||
);
|
||||
console.log(`✓ lesson (${lessonSecs}s) → ${join(out_dir, "lesson.md")}`);
|
||||
|
||||
// Seed the lesson into playbook_memory for future retrieval. Keep
|
||||
// the embedded `approach` + `context` terse per feedback_phase19_seed_text.md;
|
||||
// the rich prose lives in lesson.md and a separate `rationale` field.
|
||||
// Persist the lesson to data/_playbook_lessons/ so future scenarios
|
||||
// can read it verbatim at startup. The /vectors/playbook_memory/seed
|
||||
// endpoint rejects operations that don't match the `fill: Role xN
|
||||
// in City, ST` regex (enforced in crates/vectord/src/service.rs),
|
||||
// so embedding-based retrieval of cross-day lessons isn't wired.
|
||||
// File-based read-back is durable and explicit — future scenarios
|
||||
// pull from the lessons dir at startup and include top-N in the
|
||||
// executor's system context.
|
||||
try {
|
||||
const kinds = [...new Set(ctx.spec.events.map(e => e.kind))].join("+");
|
||||
const cities = [...new Set(ctx.spec.events.map(e => e.city))].slice(0, 3).join(",");
|
||||
await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: `cross-day-lesson-${ctx.spec.date}`,
|
||||
approach: `${kinds} day in ${cities}`,
|
||||
context: `${ctx.spec.client} ${ctx.spec.date}`,
|
||||
rationale: lesson.slice(0, 2000),
|
||||
endorsed_names: [],
|
||||
append: true,
|
||||
}),
|
||||
});
|
||||
const states = [...new Set(ctx.spec.events.map(e => e.state))].slice(0, 3).join(",");
|
||||
const lessonsDir = join("data", "_playbook_lessons");
|
||||
await mkdir(lessonsDir, { recursive: true });
|
||||
const lessonRec = {
|
||||
date: ctx.spec.date,
|
||||
client: ctx.spec.client,
|
||||
cities,
|
||||
states,
|
||||
events_total: ctx.spec.events.length,
|
||||
events_ok: ctx.results.filter(r => r.ok).length,
|
||||
checkpoint_count: checkpoints.length,
|
||||
model: OVERVIEW_MODEL,
|
||||
cloud: OVERVIEW_CLOUD,
|
||||
lesson: lesson.trim(),
|
||||
checkpoints: checkpoints.map(c => ({ after: c.after_event, risk: c.risk, hint: c.hint })),
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
const fname = `${ctx.spec.date}_${ctx.spec.client.replace(/\s+/g, "_")}_${Date.now()}.json`;
|
||||
await writeFile(join(lessonsDir, fname), JSON.stringify(lessonRec, null, 2));
|
||||
console.log(` lesson archived → ${join(lessonsDir, fname)}`);
|
||||
} catch (e) {
|
||||
console.log(` (lesson seed skipped: ${(e as Error).message})`);
|
||||
console.log(` (lesson archive skipped: ${(e as Error).message})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user