Phase 21 foundation — context stability + chunking pipeline
PRD: add Phase 20 (model matrix, wired) and Phase 21 (context stability, partial). Phase 21 exists because LLM Team hit this exact wall — running multi-model ranking on large context silently truncated, rankings degraded, no pipeline caught it. The stable answer: every agent call goes through a budget check against the model's declared context_window minus safety_margin, with a declared overflow_policy when the check fails. config/models.json: - context_window + context_budget per tier - overflow_policies block: summarize_oldest_tool_results_via_t3, chunk_lessons_via_cosine_topk, two_pass_map_reduce, escalate_to_kimi_k2_1t_or_split_decision - chunking_cache spec (data/_chunk_cache/, corpus-hash keyed) agent.ts: - estimateTokens() chars/4 biased safe ~15% - CONTEXT_WINDOWS table (fallback; prod reads models.json) - assertContextBudget() — throws on overflow with exact numbers, can bypass with bypass_budget:true for callers with their own policy - Wired into generate() and generateCloud() so EVERY call is checked scenario.ts: - T3 lesson archive to data/_playbook_lessons/*.json (the old /vectors/playbook_memory/seed path was silently failing with HTTP 400 because it requires 'fill: Role xN in City, ST' operation shape) - loadPriorLessons() at scenario start — filters by city/state match, date-sorted, takes top-3 - prior_lessons.json archived per-run (honest signal for A/B) - guidanceFor() injects up to 2 prior lessons (≤500 chars each) into the executor's per-event context - Retrospective shows explicit "Prior lessons loaded: N" line Verified: mistral correctly rejects a 150K-char prompt (7532 tokens over), gpt-oss:120b accepts it with 90K headroom. The enforcement is in-band on every call now, not an afterthought. Full chunking service (Rust) remains deferred to the sprint this feeds: crates/aibridge/src/budget.rs + chunk.rs + storaged/chunk_cache.rs
This commit is contained in:
@@ -15,6 +15,54 @@ export const SIDECAR = "http://localhost:3200";
|
||||
export const OLLAMA_CLOUD_URL = process.env.OLLAMA_CLOUD_URL ?? "https://ollama.com";
|
||||
export const OLLAMA_CLOUD_KEY = process.env.OLLAMA_CLOUD_KEY ?? "";
|
||||
|
||||
// Rough token estimator — chars/4 biased safe by ~15%. Swap to a real
|
||||
// tokenizer (tiktoken or provider endpoint) once Phase 21 lands. Good
|
||||
// enough to stop the "context silently truncated" failure mode today.
|
||||
export function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
// Known context windows — matches crates/../config/models.json. Kept in
|
||||
// code as a fallback so the test harness doesn't crash if the config is
|
||||
// missing. Production path should read from models.json.
|
||||
export const CONTEXT_WINDOWS: Record<string, number> = {
|
||||
"mistral:latest": 32768,
|
||||
"qwen2.5:latest": 32768,
|
||||
"qwen3:latest": 40960,
|
||||
"gpt-oss:20b": 131072,
|
||||
"gpt-oss:120b": 131072,
|
||||
"qwen3.5:397b": 131072,
|
||||
"kimi-k2-thinking": 200000,
|
||||
"kimi-k2:1t": 1048576,
|
||||
"deepseek-v3.1:671b": 131072,
|
||||
"glm-4.7": 131072,
|
||||
};
|
||||
const DEFAULT_CONTEXT_WINDOW = 32768;
|
||||
const DEFAULT_SAFETY_MARGIN = 2000;
|
||||
|
||||
// Fail LOUDLY if a prompt would blow the model's context. The whole
|
||||
// point of Phase 21 is to stop silent truncation — so we throw with the
|
||||
// numbers. Callers that expect to handle overflow should chunk BEFORE
|
||||
// calling; they can also set bypass: true to opt out (T5 gatekeeper
|
||||
// handles its own overflow policy).
|
||||
export function assertContextBudget(
|
||||
model: string,
|
||||
prompt: string,
|
||||
opts: { system?: string; max_tokens?: number; safety_margin?: number; bypass?: boolean } = {}
|
||||
): { estimated: number; window: number; remaining: number } {
|
||||
const window = CONTEXT_WINDOWS[model] ?? DEFAULT_CONTEXT_WINDOW;
|
||||
const safety = opts.safety_margin ?? DEFAULT_SAFETY_MARGIN;
|
||||
const estimated = estimateTokens(prompt) + estimateTokens(opts.system ?? "") + (opts.max_tokens ?? 800);
|
||||
const remaining = window - estimated - safety;
|
||||
if (remaining < 0 && !opts.bypass) {
|
||||
throw new Error(
|
||||
`context overflow: model=${model} est=${estimated}t window=${window}t safety=${safety}t over=${-remaining}t. ` +
|
||||
`Chunk the prompt (see config/models.json overflow_policies) or set bypass:true if you know the risk.`
|
||||
);
|
||||
}
|
||||
return { estimated, window, remaining };
|
||||
}
|
||||
|
||||
// --- Shared types ---
|
||||
|
||||
export type Role = "executor" | "reviewer";
|
||||
@@ -101,7 +149,13 @@ export async function generate(model: string, prompt: string, opts: {
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
system?: string;
|
||||
bypass_budget?: boolean;
|
||||
} = {}): Promise<string> {
|
||||
assertContextBudget(model, prompt, {
|
||||
system: opts.system,
|
||||
max_tokens: opts.max_tokens,
|
||||
bypass: opts.bypass_budget,
|
||||
});
|
||||
const body: Record<string, any> = {
|
||||
model,
|
||||
prompt,
|
||||
@@ -126,10 +180,16 @@ export async function generateCloud(model: string, prompt: string, opts: {
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
system?: string;
|
||||
bypass_budget?: boolean;
|
||||
} = {}): Promise<string> {
|
||||
if (!OLLAMA_CLOUD_KEY) {
|
||||
throw new Error("OLLAMA_CLOUD_KEY not set; cannot reach Ollama Cloud");
|
||||
}
|
||||
assertContextBudget(model, prompt, {
|
||||
system: opts.system,
|
||||
max_tokens: opts.max_tokens,
|
||||
bypass: opts.bypass_budget,
|
||||
});
|
||||
const body: Record<string, any> = {
|
||||
model,
|
||||
prompt,
|
||||
|
||||
Reference in New Issue
Block a user