checkpoint: matrix-agent-validated (2026-04-25)
Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.
WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.
WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
* UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
* REVISE: chains versions, parent.superseded_at + superseded_by stamped
* RETIRE: marks specific trace retired with reason, excluded from retrieval
* HISTORY: walks chain root→tip, cycle-safe
KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces
Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
profit
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env bun
|
||||
// Autonomous scrum loop — wraps scrum_master_pipeline.ts + scrum_applier.ts
|
||||
// in a goal-driven retry loop. Observer is POSTed an iteration summary at
|
||||
// every boundary so it can build meta-commentary outside the loop's epistemic
|
||||
// scope.
|
||||
//
|
||||
// Usage:
|
||||
// LOOP_TARGETS="crates/a/src/x.rs,crates/b/src/y.rs" \
|
||||
// LOOP_MAX_ITERS=5 \
|
||||
// LOOP_PUSH=1 \
|
||||
// bun run tests/real-world/autonomous_loop.ts
|
||||
//
|
||||
// Stop conditions: max_iters reached OR 2 consecutive iters with 0 commits.
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { appendFile, readFile } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
|
||||
const REPO = "/home/profit/lakehouse";
|
||||
const OBSERVER = process.env.LOOP_OBSERVER ?? "http://localhost:3800";
|
||||
const BRANCH = process.env.LOOP_BRANCH ?? "scrum/auto-apply-19814";
|
||||
const MAX_ITERS = Number(process.env.LOOP_MAX_ITERS ?? 3);
|
||||
const PUSH = process.env.LOOP_PUSH === "1";
|
||||
const MIN_CONF = process.env.LOOP_MIN_CONF ?? "85";
|
||||
// Optional override — when unset, let scrum_applier.ts use ITS default
|
||||
// (currently x-ai/grok-4.1-fast on openrouter). The prior hardcoded
|
||||
// qwen3-coder:480b default was clobbering the applier's own default
|
||||
// and forcing every iter to hit the throttled ollama_cloud account.
|
||||
const APPLIER_MODEL = process.env.LOOP_APPLIER_MODEL;
|
||||
const APPLIER_PROVIDER = process.env.LOOP_APPLIER_PROVIDER;
|
||||
const TARGETS = (process.env.LOOP_TARGETS ?? "crates/queryd/src/service.rs,crates/gateway/src/main.rs,crates/gateway/src/v1/mod.rs")
|
||||
.split(",").map(s => s.trim()).filter(Boolean);
|
||||
|
||||
const FORENSIC = process.env.LH_SCRUM_FORENSIC ?? `${REPO}/docs/SCRUM_FORENSIC_PROMPT.md`;
|
||||
const PROPOSAL = process.env.LH_SCRUM_PROPOSAL ?? `${REPO}/docs/SCRUM_FIX_WAVE.md`;
|
||||
|
||||
const LOOP_ID = `loop_${Date.now().toString(36)}`;
|
||||
const JOURNAL = `${REPO}/data/_kb/autonomous_loops.jsonl`;
|
||||
|
||||
interface IterResult {
|
||||
iter: number;
|
||||
scrum_reviews_added: number;
|
||||
applier_outcomes: Record<string, number>;
|
||||
commits_landed: number;
|
||||
commit_shas: string[];
|
||||
build_status: "green" | "red" | "unknown";
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
function log(msg: string) {
|
||||
const ts = new Date().toISOString().slice(11, 19);
|
||||
console.log(`[loop ${LOOP_ID} ${ts}] ${msg}`);
|
||||
}
|
||||
|
||||
function runCmd(cmd: string, args: string[], env: Record<string, string> = {}): Promise<{ code: number; stdout: string; stderr: string }> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(cmd, args, { cwd: REPO, env: { ...process.env, ...env } });
|
||||
let stdout = "", stderr = "";
|
||||
child.stdout.on("data", (d) => { stdout += d; process.stdout.write(d); });
|
||||
child.stderr.on("data", (d) => { stderr += d; process.stderr.write(d); });
|
||||
child.on("close", (code) => resolve({ code: code ?? -1, stdout, stderr }));
|
||||
});
|
||||
}
|
||||
|
||||
async function countLines(path: string): Promise<number> {
|
||||
if (!existsSync(path)) return 0;
|
||||
const text = await readFile(path, "utf8");
|
||||
return text.split("\n").filter(Boolean).length;
|
||||
}
|
||||
|
||||
async function gitHeadSha(): Promise<string> {
|
||||
const r = await runCmd("git", ["rev-parse", "HEAD"]);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
async function commitsSince(baseSha: string): Promise<string[]> {
|
||||
const r = await runCmd("git", ["log", "--oneline", `${baseSha}..HEAD`]);
|
||||
return r.stdout.trim().split("\n").filter(Boolean);
|
||||
}
|
||||
|
||||
async function cargoCheckGreen(): Promise<boolean> {
|
||||
log("cargo check --workspace …");
|
||||
const r = await runCmd("cargo", ["check", "--workspace", "--quiet"]);
|
||||
return r.code === 0;
|
||||
}
|
||||
|
||||
async function postObserver(payload: object) {
|
||||
try {
|
||||
const r = await fetch(`${OBSERVER}/event`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
if (!r.ok) log(`observer POST returned ${r.status}`);
|
||||
} catch (e: any) {
|
||||
log(`observer POST failed: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function runIter(iter: number, baseSha: string): Promise<IterResult> {
|
||||
const t0 = Date.now();
|
||||
log(`══ iter ${iter} start (base ${baseSha.slice(0, 8)}) targets=${TARGETS.length}`);
|
||||
|
||||
const reviewsBefore = await countLines(`${REPO}/data/_kb/scrum_reviews.jsonl`);
|
||||
const applyBefore = await countLines(`${REPO}/data/_kb/auto_apply.jsonl`);
|
||||
|
||||
log(`scrum_master_pipeline.ts → ${TARGETS.length} files`);
|
||||
await runCmd("bun", ["run", "tests/real-world/scrum_master_pipeline.ts"], {
|
||||
LH_SCRUM_FILES: TARGETS.join(","),
|
||||
LH_SCRUM_FORENSIC: FORENSIC,
|
||||
LH_SCRUM_PROPOSAL: PROPOSAL,
|
||||
});
|
||||
|
||||
log(`scrum_applier.ts COMMIT=1 MIN_CONF=${MIN_CONF} files=${TARGETS.length}`);
|
||||
// Only forward model/provider when explicitly overridden — otherwise
|
||||
// let scrum_applier.ts use its own defaults (Grok 4.1 fast on openrouter).
|
||||
const applierEnv: Record<string, string> = {
|
||||
LH_APPLIER_COMMIT: "1",
|
||||
LH_APPLIER_MIN_CONF: MIN_CONF,
|
||||
LH_APPLIER_MAX_FILES: String(TARGETS.length),
|
||||
LH_APPLIER_BRANCH: BRANCH,
|
||||
// Constrain applier to THIS iter's targets so it patches what we
|
||||
// just reviewed instead of the highest-confidence file from history.
|
||||
LH_APPLIER_FILES: TARGETS.join(","),
|
||||
};
|
||||
if (APPLIER_MODEL) applierEnv.LH_APPLIER_MODEL = APPLIER_MODEL;
|
||||
if (APPLIER_PROVIDER) applierEnv.LH_APPLIER_PROVIDER = APPLIER_PROVIDER;
|
||||
await runCmd("bun", ["run", "tests/real-world/scrum_applier.ts"], applierEnv);
|
||||
|
||||
const reviewsAfter = await countLines(`${REPO}/data/_kb/scrum_reviews.jsonl`);
|
||||
const applyAfterText = existsSync(`${REPO}/data/_kb/auto_apply.jsonl`)
|
||||
? await readFile(`${REPO}/data/_kb/auto_apply.jsonl`, "utf8")
|
||||
: "";
|
||||
const applyRows = applyAfterText.split("\n").filter(Boolean).slice(applyBefore);
|
||||
const outcomes: Record<string, number> = {};
|
||||
for (const line of applyRows) {
|
||||
try {
|
||||
const o = JSON.parse(line);
|
||||
outcomes[o.action ?? "?"] = (outcomes[o.action ?? "?"] ?? 0) + 1;
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
|
||||
const commitShas = await commitsSince(baseSha);
|
||||
const buildStatus = commitShas.length > 0 ? (await cargoCheckGreen() ? "green" : "red") : "unknown";
|
||||
|
||||
const result: IterResult = {
|
||||
iter,
|
||||
scrum_reviews_added: reviewsAfter - reviewsBefore,
|
||||
applier_outcomes: outcomes,
|
||||
commits_landed: commitShas.length,
|
||||
commit_shas: commitShas.map(s => s.split(" ")[0]),
|
||||
build_status: buildStatus,
|
||||
duration_ms: Date.now() - t0,
|
||||
};
|
||||
|
||||
log(`iter ${iter} done — reviews+${result.scrum_reviews_added} commits=${result.commits_landed} build=${buildStatus} (${(result.duration_ms / 1000).toFixed(1)}s)`);
|
||||
|
||||
await postObserver({
|
||||
source: "autonomous_loop",
|
||||
loop_id: LOOP_ID,
|
||||
event_kind: "iteration_complete",
|
||||
iter,
|
||||
targets: TARGETS,
|
||||
success: buildStatus !== "red",
|
||||
scrum_reviews_added: result.scrum_reviews_added,
|
||||
applier_outcomes: result.applier_outcomes,
|
||||
commits_landed: result.commits_landed,
|
||||
commit_shas: result.commit_shas,
|
||||
build_status: buildStatus,
|
||||
duration_ms: result.duration_ms,
|
||||
ts: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await appendFile(JOURNAL, JSON.stringify({ loop_id: LOOP_ID, ...result, ts: new Date().toISOString() }) + "\n");
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log(`autonomous loop starting · branch=${BRANCH} max_iters=${MAX_ITERS} push=${PUSH}`);
|
||||
log(`targets: ${TARGETS.join(", ")}`);
|
||||
|
||||
const branchR = await runCmd("git", ["branch", "--show-current"]);
|
||||
if (branchR.stdout.trim() !== BRANCH) {
|
||||
log(`ERROR: on branch ${branchR.stdout.trim()}, expected ${BRANCH}. Refusing to run.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let consecutiveZero = 0;
|
||||
for (let iter = 1; iter <= MAX_ITERS; iter++) {
|
||||
const baseSha = await gitHeadSha();
|
||||
const result = await runIter(iter, baseSha);
|
||||
|
||||
if (PUSH && result.commits_landed > 0) {
|
||||
log(`git push origin ${BRANCH}`);
|
||||
const pushR = await runCmd("git", ["push", "origin", BRANCH]);
|
||||
if (pushR.code !== 0) log(`push failed (continuing): ${pushR.stderr.slice(0, 200)}`);
|
||||
}
|
||||
|
||||
consecutiveZero = result.commits_landed === 0 ? consecutiveZero + 1 : 0;
|
||||
if (consecutiveZero >= 2) {
|
||||
log(`STOP: 2 consecutive iters with 0 commits. Loop converged or stuck.`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
log(`loop ${LOOP_ID} complete. Journal: ${JOURNAL}`);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
log(`FATAL: ${e.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,528 @@
|
||||
// Real-world architecture stress test — 6 iterations of the full pipeline
|
||||
// against the PRD as a corpus. Goal: prove at scale what Phase 21
|
||||
// promised (context continuation + tree-split), plus Phase 19
|
||||
// compounding across iterations.
|
||||
//
|
||||
// Run: bun run tests/real-world/enrich_prd_pipeline.ts
|
||||
//
|
||||
// No mocks. No skipped layers. On any error, the test triggers
|
||||
// cloud-rescue rather than fail — it's the architecture's job to
|
||||
// recover. The test FAILS only if we can't complete 6 iterations.
|
||||
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
const PRD_PATH = "/home/profit/lakehouse/docs/PRD.md";
|
||||
const CHUNK_SIZE = 800; // chars per chunk — ~200 tokens
|
||||
const CHUNK_OVERLAP = 120;
|
||||
const TOP_K_RETRIEVE = 12; // chunks per iteration — pulled up to force overflow
|
||||
const CONTEXT_BUDGET_CHARS = 4000; // tight budget — forces tree-split on every iter
|
||||
const INJECT_FAIL_ON_ITER = 3; // force the TASK-retry loop on iter 3
|
||||
|
||||
// Continuation controls (per-cloud-call) — used for output-overflow.
|
||||
// Separate from the task-retry loop (per-task) — that handles errors
|
||||
// across attempts.
|
||||
const PRIMARY_MAX_TOKENS = 150; // tight — forces truncation
|
||||
const CONTINUATION_MAX_TOKENS = 300; // each continuation doubles headroom
|
||||
const MAX_CONTINUATIONS = 6; // max stitch pieces per cloud call
|
||||
|
||||
// Task-level retry loop (J's clarification, 2026-04-22):
|
||||
// When a TASK errors, retry the whole task up to 6 times. Each
|
||||
// retry gets prior attempts' failures injected as learning context,
|
||||
// so attempt N+1 is informed by what N failed at. The loop caps at
|
||||
// 6 to avoid infinite spinning on genuinely unsolvable tasks.
|
||||
const MAX_TASK_RETRIES = 6;
|
||||
|
||||
// To FORCE the retry loop on iter INJECT_FAIL_ON_ITER, cycle through
|
||||
// 5 deliberately-invalid models + 1 valid one. Attempts 1-5 will
|
||||
// 502/404 from Ollama Cloud; attempt 6 finally succeeds. Proves the
|
||||
// loop fires all 6 with compounding failure context.
|
||||
const FORCE_RETRY_MODEL_SEQUENCE = [
|
||||
"deliberately-invalid-model-attempt-1",
|
||||
"deliberately-invalid-model-attempt-2",
|
||||
"deliberately-invalid-model-attempt-3",
|
||||
"deliberately-invalid-model-attempt-4",
|
||||
"deliberately-invalid-model-attempt-5",
|
||||
"gpt-oss:20b", // 6th attempt succeeds
|
||||
];
|
||||
const GATEWAY = "http://localhost:3100";
|
||||
const SIDECAR = "http://localhost:3200";
|
||||
const CLOUD_MODEL = "gpt-oss:120b";
|
||||
const RESCUE_MODEL = "gpt-oss:20b"; // fallback local cloud model via sidecar
|
||||
const RUN_NONCE = Date.now().toString(36);
|
||||
const OUT_DIR = `/home/profit/lakehouse/tests/real-world/runs/${RUN_NONCE}`;
|
||||
|
||||
// The 6 progressively-compounding questions. #6 explicitly requires
|
||||
// synthesis across prior 5 answers.
|
||||
const QUESTIONS: string[] = [
|
||||
"Summarize the Lakehouse project's one-paragraph thesis: what problem does it solve, what's the unique approach?",
|
||||
"How does Phase 19 playbook memory turn successful fills into a signal that boosts future rankings?",
|
||||
"Explain the role of Phase 24 observer in the learning loop — what does it see, and what does it feed into?",
|
||||
"What's the VRAM-aware profile swap mechanism in Phase 17, and why does it matter for multi-model serving?",
|
||||
"How do Phase 25 validity windows and Phase 27 playbook versioning interact when a schema drifts?",
|
||||
"Synthesize the prior 5 answers: how do the pieces (playbook memory, observer, profile swap, validity windows, versioning) compose into a system that measurably gets smarter over time? Cite specific prior answers.",
|
||||
];
|
||||
|
||||
type Chunk = { id: string; text: string; embedding: number[]; offset: number };
|
||||
|
||||
interface IterationResult {
|
||||
iteration: number;
|
||||
question: string;
|
||||
retrieval_top_k: number;
|
||||
context_chars_before_budget: number;
|
||||
tree_split_fired: boolean;
|
||||
cloud_calls_total: number;
|
||||
continuation_retries: number;
|
||||
rescue_triggered: boolean;
|
||||
// Task-level retry telemetry
|
||||
task_attempts_made: number; // how many attempts fired (1 = first succeeded)
|
||||
task_retry_history: Array<{ n: number; model: string; error: string }>;
|
||||
playbook_id: string | null;
|
||||
tokens_prompt: number;
|
||||
tokens_completion: number;
|
||||
citations_from_prior_iterations: string[];
|
||||
duration_ms: number;
|
||||
answer_preview: string;
|
||||
errors_recovered: string[];
|
||||
}
|
||||
|
||||
function log(msg: string) { console.log(`[enrich] ${msg}`); }
|
||||
|
||||
function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
|
||||
|
||||
function cosine(a: number[], b: number[]): number {
|
||||
let dot = 0, na = 0, nb = 0;
|
||||
for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
|
||||
if (na === 0 || nb === 0) return 0;
|
||||
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
||||
}
|
||||
|
||||
function hash(s: string): string { return createHash("sha256").update(s).digest("hex").slice(0, 10); }
|
||||
|
||||
async function embedBatch(texts: string[]): Promise<number[][]> {
|
||||
// Sidecar /embed accepts a list. On partial failure, retry individually.
|
||||
const r = await fetch(`${SIDECAR}/embed`, {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ texts }),
|
||||
signal: AbortSignal.timeout(120000),
|
||||
});
|
||||
if (!r.ok) throw new Error(`embed batch ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
return j.embeddings;
|
||||
}
|
||||
|
||||
function chunkText(text: string): Array<{ text: string; offset: number }> {
|
||||
const out: Array<{ text: string; offset: number }> = [];
|
||||
let i = 0;
|
||||
while (i < text.length) {
|
||||
const end = Math.min(i + CHUNK_SIZE, text.length);
|
||||
const slice = text.slice(i, end).trim();
|
||||
if (slice.length > 50) out.push({ text: slice, offset: i });
|
||||
if (end >= text.length) break;
|
||||
i = end - CHUNK_OVERLAP;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function chat(opts: {
|
||||
provider: "ollama" | "ollama_cloud",
|
||||
model: string,
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
max_tokens: number,
|
||||
think: boolean,
|
||||
}): Promise<{ content: string; prompt_tokens: number; completion_tokens: number; finish_reason: string }> {
|
||||
const r = await fetch(`${GATEWAY}/v1/chat`, {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ ...opts }),
|
||||
signal: AbortSignal.timeout(180000),
|
||||
});
|
||||
if (!r.ok) throw new Error(`/v1/chat ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
return {
|
||||
content: j.choices?.[0]?.message?.content ?? "",
|
||||
prompt_tokens: j.usage?.prompt_tokens ?? 0,
|
||||
completion_tokens: j.usage?.completion_tokens ?? 0,
|
||||
finish_reason: j.choices?.[0]?.finish_reason ?? "?",
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Tree-split over oversized chunk set ──────────────────────────
|
||||
async function treeSplitSummarize(
|
||||
chunks: Chunk[],
|
||||
question: string,
|
||||
): Promise<{ scratchpad: string; cloud_calls: number }> {
|
||||
// Shard into groups fitting within half the budget each.
|
||||
const perShard = Math.max(1, Math.floor((CONTEXT_BUDGET_CHARS / 2) / CHUNK_SIZE));
|
||||
const shards: Chunk[][] = [];
|
||||
for (let i = 0; i < chunks.length; i += perShard) {
|
||||
shards.push(chunks.slice(i, i + perShard));
|
||||
}
|
||||
log(` tree-split: ${chunks.length} chunks → ${shards.length} shards of up to ${perShard}`);
|
||||
let scratchpad = "";
|
||||
let cloud_calls = 0;
|
||||
for (let si = 0; si < shards.length; si++) {
|
||||
const shard = shards[si];
|
||||
const shardText = shard.map(c => `[chunk @${c.offset}]\n${c.text}`).join("\n\n");
|
||||
const userMsg = `Question: ${question}\n\nShard ${si + 1}/${shards.length} of source material:\n\n${shardText}\n\nScratchpad so far:\n${scratchpad || "(empty)"}\n\nUpdate the scratchpad: extract only facts from THIS shard that help answer the question. Be terse. No prose.`;
|
||||
const r = await chat({
|
||||
provider: "ollama_cloud",
|
||||
model: CLOUD_MODEL,
|
||||
messages: [
|
||||
{ role: "system", content: "You maintain a concise factual scratchpad across multiple shards of source text. No prose outside the scratchpad. Each shard, append ≤80 words of relevant facts." },
|
||||
{ role: "user", content: userMsg },
|
||||
],
|
||||
max_tokens: 500,
|
||||
think: false,
|
||||
});
|
||||
cloud_calls += 1;
|
||||
scratchpad += `\n--- shard ${si + 1} notes ---\n${r.content.trim()}`;
|
||||
if (scratchpad.length > CONTEXT_BUDGET_CHARS) {
|
||||
// truncate oldest halves
|
||||
scratchpad = scratchpad.slice(-CONTEXT_BUDGET_CHARS);
|
||||
log(` tree-split: scratchpad truncated to ${scratchpad.length} chars`);
|
||||
}
|
||||
}
|
||||
return { scratchpad, cloud_calls };
|
||||
}
|
||||
|
||||
// ─── Continuable generate — up to max_continuations stitches ──────
|
||||
//
|
||||
// Two failure modes handled:
|
||||
// A) Empty response — typically thinking model burned the budget
|
||||
// on hidden reasoning. Retry with 2× max_tokens.
|
||||
// B) Truncated response (finish_reason=length) — answer got cut off
|
||||
// mid-sentence. Pass the partial back as scratchpad and ask the
|
||||
// model to continue from where it stopped.
|
||||
//
|
||||
// Stitching: keep appending content across retries; prompt_tokens and
|
||||
// completion_tokens accumulate; finish_reason reflects the LAST call.
|
||||
// Loop exits on the first call that finishes cleanly (stop) with
|
||||
// non-empty content, OR when retries hit the cap.
|
||||
async function generateContinuable(
|
||||
opts: Parameters<typeof chat>[0] & { max_continuations?: number },
|
||||
): Promise<{ content: string; prompt_tokens: number; completion_tokens: number; continuation_retries: number; finish_reason: string }> {
|
||||
const maxCont = opts.max_continuations ?? 1;
|
||||
let total = await chat(opts);
|
||||
let retries = 0;
|
||||
while (retries < maxCont && (total.content.length === 0 || total.finish_reason === "length")) {
|
||||
retries += 1;
|
||||
const mode = total.content.length === 0 ? "empty" : "truncated";
|
||||
log(` continuation retry ${retries}/${maxCont} (${mode}: finish=${total.finish_reason}, content=${total.content.length} chars)`);
|
||||
// Continuation prompt — branch on failure mode:
|
||||
// empty → retry with 2× tokens, same prompt (thinking budget)
|
||||
// length → pass the partial as assistant turn, ask to continue
|
||||
const continuationMessages = total.content.length === 0
|
||||
? opts.messages
|
||||
: [
|
||||
...opts.messages,
|
||||
{ role: "assistant", content: total.content },
|
||||
{ role: "user", content: "Continue from exactly where you stopped. Do not repeat. Finish the answer." },
|
||||
];
|
||||
const continued = await chat({
|
||||
...opts,
|
||||
max_tokens: CONTINUATION_MAX_TOKENS,
|
||||
messages: continuationMessages,
|
||||
});
|
||||
total = {
|
||||
content: total.content + continued.content,
|
||||
prompt_tokens: total.prompt_tokens + continued.prompt_tokens,
|
||||
completion_tokens: total.completion_tokens + continued.completion_tokens,
|
||||
finish_reason: continued.finish_reason,
|
||||
};
|
||||
}
|
||||
return { ...total, continuation_retries: retries };
|
||||
}
|
||||
|
||||
// ─── Single iteration: retrieve → budget-check → chat → seed ─────
|
||||
async function runIteration(
|
||||
iteration: number,
|
||||
question: string,
|
||||
allChunks: Chunk[],
|
||||
priorPlaybookIds: string[],
|
||||
priorAnswers: string[],
|
||||
): Promise<IterationResult> {
|
||||
const started = Date.now();
|
||||
const errorsRecovered: string[] = [];
|
||||
log(`iter ${iteration}: "${question.slice(0, 70)}..."`);
|
||||
|
||||
// 1. Embed the question
|
||||
const qEmb = (await embedBatch([question]))[0];
|
||||
|
||||
// 2. Retrieve top-K chunks by cosine
|
||||
const scored = allChunks
|
||||
.map(c => ({ c, score: cosine(qEmb, c.embedding) }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, TOP_K_RETRIEVE);
|
||||
const chunks = scored.map(x => x.c);
|
||||
log(` retrieved top ${chunks.length} chunks (score range ${scored[0].score.toFixed(3)} .. ${scored[scored.length - 1].score.toFixed(3)})`);
|
||||
|
||||
// 3. Context budget check — tree-split if over
|
||||
const contextChars = chunks.map(c => c.text).join("\n\n").length;
|
||||
let contextForPrompt: string;
|
||||
let treeSplit = false;
|
||||
let cloudCallsTotal = 0;
|
||||
if (contextChars > CONTEXT_BUDGET_CHARS) {
|
||||
treeSplit = true;
|
||||
log(` context ${contextChars} chars > budget ${CONTEXT_BUDGET_CHARS} → tree-split`);
|
||||
const { scratchpad, cloud_calls } = await treeSplitSummarize(chunks, question);
|
||||
contextForPrompt = `Distilled scratchpad from ${chunks.length} source chunks (too large to fit directly):\n${scratchpad}`;
|
||||
cloudCallsTotal += cloud_calls;
|
||||
} else {
|
||||
contextForPrompt = chunks.map(c => `[chunk @${c.offset}]\n${c.text}`).join("\n\n");
|
||||
}
|
||||
|
||||
// 4. Seed prompt with prior iteration answers (real compounding).
|
||||
// Not just IDs — the model needs the CONTENT to synthesize.
|
||||
let citationBlock = "";
|
||||
let citationsReceived: string[] = [];
|
||||
if (priorPlaybookIds.length > 0 && priorAnswers.length > 0) {
|
||||
const lines = priorAnswers.map((ans, i) => {
|
||||
const pid = priorPlaybookIds[i]?.slice(0, 12) ?? "unknown";
|
||||
// Trim each prior answer to ~400 chars so we don't blow budget
|
||||
return `[pb:${pid}] iter ${i + 1} answer:\n${ans.slice(0, 400)}\n`;
|
||||
});
|
||||
citationBlock = `\n\n═══ PRIOR ITERATIONS (compounding context) ═══\n${lines.join("\n")}═══ end prior iterations ═══\n\nYour answer MUST cite specific prior iterations using [pb:ID] notation when drawing on them. Synthesis questions require explicit cross-iteration reasoning.`;
|
||||
citationsReceived = priorPlaybookIds.slice();
|
||||
}
|
||||
|
||||
// 5. TASK-LEVEL RETRY LOOP — per J's clarification 2026-04-22.
|
||||
// Try the task up to MAX_TASK_RETRIES times. Each retry:
|
||||
// a) Picks a model (normally CLOUD_MODEL; on INJECT_FAIL_ON_ITER,
|
||||
// cycles through 5 invalid models + 1 valid to force full loop)
|
||||
// b) Injects prior attempt errors as learning context
|
||||
// c) If the attempt succeeds (non-empty, >100 chars), loop exits
|
||||
// d) Otherwise, records failure and tries again with the learning
|
||||
//
|
||||
// Cap at 6 so we don't spin forever on unsolvable tasks.
|
||||
let result: { content: string; prompt_tokens: number; completion_tokens: number; continuation_retries: number; finish_reason: string } | null = null;
|
||||
let rescueTriggered = false;
|
||||
const taskAttemptHistory: Array<{ n: number; model: string; error: string }> = [];
|
||||
const forceRetries = iteration === INJECT_FAIL_ON_ITER;
|
||||
if (forceRetries) log(` FORCING TASK-RETRY LOOP — iter ${iteration} will cycle through 5 invalid models + 1 valid`);
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_TASK_RETRIES; attempt++) {
|
||||
const modelForAttempt = forceRetries
|
||||
? FORCE_RETRY_MODEL_SEQUENCE[attempt - 1]
|
||||
: CLOUD_MODEL;
|
||||
// Compose a prior-attempts learning block for attempts 2+
|
||||
const learningBlock = taskAttemptHistory.length > 0
|
||||
? `\n\n═══ PRIOR ATTEMPTS THIS TASK (do NOT repeat these failures; adjust approach) ═══\n${taskAttemptHistory.map(a => `Attempt ${a.n} (model ${a.model}) failed: ${a.error.slice(0, 160)}`).join("\n")}\n═══ end prior attempts ═══\n`
|
||||
: "";
|
||||
log(` task attempt ${attempt}/${MAX_TASK_RETRIES}: model=${modelForAttempt}${learningBlock ? " [with prior-failure context]" : ""}`);
|
||||
try {
|
||||
const r = await generateContinuable({
|
||||
provider: "ollama_cloud",
|
||||
model: modelForAttempt,
|
||||
messages: [
|
||||
{ role: "system", content: "You answer questions about the Lakehouse PRD using only the provided source material and prior iteration answers. Be specific. Cite chunk offsets OR [pb:ID] markers. Write a detailed 250-word answer." },
|
||||
{ role: "user", content: `Question: ${question}\n\nSource material:\n${contextForPrompt}${citationBlock}${learningBlock}` },
|
||||
],
|
||||
max_tokens: PRIMARY_MAX_TOKENS,
|
||||
think: false,
|
||||
max_continuations: MAX_CONTINUATIONS,
|
||||
});
|
||||
cloudCallsTotal += 1 + r.continuation_retries;
|
||||
if (r.content && r.content.length > 100) {
|
||||
// Acceptable answer — exit loop
|
||||
result = r;
|
||||
if (attempt > 1) {
|
||||
log(` task attempt ${attempt} SUCCEEDED (${r.content.length} chars) after ${attempt - 1} prior failures`);
|
||||
rescueTriggered = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
// Thin response — count as failure with learning signal
|
||||
const err = `thin-answer: ${r.content.length} chars, finish=${r.finish_reason}`;
|
||||
taskAttemptHistory.push({ n: attempt, model: modelForAttempt, error: err });
|
||||
errorsRecovered.push(`attempt ${attempt}: ${err}`);
|
||||
} catch (e) {
|
||||
const err = (e as Error).message;
|
||||
taskAttemptHistory.push({ n: attempt, model: modelForAttempt, error: err });
|
||||
errorsRecovered.push(`attempt ${attempt}: ${err.slice(0, 120)}`);
|
||||
cloudCallsTotal += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Last-ditch: if all 6 task attempts failed, try the local fallback
|
||||
// once more so we at least return SOMETHING. This is the "don't get
|
||||
// caught in a loop, accept best-so-far" rule J stated explicitly.
|
||||
if (!result) {
|
||||
errorsRecovered.push(`all ${MAX_TASK_RETRIES} task attempts failed — local fallback`);
|
||||
rescueTriggered = true;
|
||||
try {
|
||||
result = await generateContinuable({
|
||||
provider: "ollama",
|
||||
model: "qwen3.5:latest",
|
||||
messages: [{ role: "user", content: `Q: ${question}\n\n${contextForPrompt.slice(0, 4000)}` }],
|
||||
max_tokens: 300,
|
||||
think: false,
|
||||
max_continuations: 2,
|
||||
});
|
||||
cloudCallsTotal += 1 + result.continuation_retries;
|
||||
} catch (e) {
|
||||
// Absolute last resort — fabricate a skeleton result
|
||||
result = {
|
||||
content: `[task failed after ${MAX_TASK_RETRIES} retries + local fallback: ${(e as Error).message}]`,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
continuation_retries: 0,
|
||||
finish_reason: "error",
|
||||
};
|
||||
}
|
||||
}
|
||||
if (result.content.length === 0) {
|
||||
errorsRecovered.push("even rescue returned empty — last-ditch local fallback");
|
||||
rescueTriggered = true;
|
||||
result = await generateContinuable({
|
||||
provider: "ollama",
|
||||
model: "qwen3.5:latest",
|
||||
messages: [{ role: "user", content: `Q: ${question}\n\n${contextForPrompt.slice(0, 4000)}` }],
|
||||
max_tokens: 300,
|
||||
think: false,
|
||||
});
|
||||
cloudCallsTotal += 1;
|
||||
}
|
||||
|
||||
// 6. Seed playbook with the answer
|
||||
let playbook_id: string | null = null;
|
||||
try {
|
||||
const ts = new Date().toISOString();
|
||||
const seedOp = `TEST: enrich_prd_run_${RUN_NONCE} iter${iteration} in Corpus, PRD`;
|
||||
const r = await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: seedOp,
|
||||
approach: `q="${question.slice(0, 80)}" context_chars=${contextChars} tree_split=${treeSplit}`,
|
||||
context: result.content.slice(0, 600),
|
||||
endorsed_names: [`iter${iteration}_${RUN_NONCE}`],
|
||||
append: true,
|
||||
}),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (r.ok) {
|
||||
const j: any = await r.json();
|
||||
playbook_id = j.outcome?.playbook_id ?? null;
|
||||
} else {
|
||||
errorsRecovered.push(`seed ${r.status}: ${(await r.text()).slice(0, 100)}`);
|
||||
}
|
||||
} catch (e) {
|
||||
errorsRecovered.push(`seed exception: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
iteration,
|
||||
question,
|
||||
retrieval_top_k: chunks.length,
|
||||
context_chars_before_budget: contextChars,
|
||||
tree_split_fired: treeSplit,
|
||||
cloud_calls_total: cloudCallsTotal,
|
||||
continuation_retries: result.continuation_retries,
|
||||
rescue_triggered: rescueTriggered,
|
||||
task_attempts_made: taskAttemptHistory.length + 1, // +1 for the successful attempt
|
||||
task_retry_history: taskAttemptHistory,
|
||||
playbook_id,
|
||||
tokens_prompt: result.prompt_tokens,
|
||||
tokens_completion: result.completion_tokens,
|
||||
citations_from_prior_iterations: citationsReceived,
|
||||
duration_ms: Date.now() - started,
|
||||
answer_preview: result.content.slice(0, 500),
|
||||
errors_recovered: errorsRecovered,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(OUT_DIR, { recursive: true });
|
||||
log(`run nonce: ${RUN_NONCE}`);
|
||||
log(`output dir: ${OUT_DIR}`);
|
||||
|
||||
// ─── Phase 1: load, chunk, embed the PRD ───────────────────────
|
||||
log(`loading PRD from ${PRD_PATH}`);
|
||||
const prd = await readFile(PRD_PATH, "utf8");
|
||||
log(`PRD: ${prd.length} chars, ${prd.split("\n").length} lines`);
|
||||
|
||||
const raw_chunks = chunkText(prd);
|
||||
log(`chunked into ${raw_chunks.length} pieces (size ${CHUNK_SIZE}, overlap ${CHUNK_OVERLAP})`);
|
||||
|
||||
// Embed in batches of 32 to avoid sidecar overload
|
||||
const allChunks: Chunk[] = [];
|
||||
const BATCH = 32;
|
||||
const t0 = Date.now();
|
||||
for (let i = 0; i < raw_chunks.length; i += BATCH) {
|
||||
const batch = raw_chunks.slice(i, i + BATCH);
|
||||
const embs = await embedBatch(batch.map(b => b.text));
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
allChunks.push({
|
||||
id: hash(batch[j].text),
|
||||
text: batch[j].text,
|
||||
embedding: embs[j].map(x => Number(x)),
|
||||
offset: batch[j].offset,
|
||||
});
|
||||
}
|
||||
log(` embedded ${allChunks.length}/${raw_chunks.length}`);
|
||||
}
|
||||
log(`embedded all ${allChunks.length} chunks in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
||||
|
||||
// ─── Phase 2: 6 iterations ─────────────────────────────────────
|
||||
const results: IterationResult[] = [];
|
||||
const priorIds: string[] = [];
|
||||
const priorAnswers: string[] = [];
|
||||
for (let i = 1; i <= QUESTIONS.length; i++) {
|
||||
const q = QUESTIONS[i - 1];
|
||||
const r = await runIteration(i, q, allChunks, priorIds, priorAnswers);
|
||||
results.push(r);
|
||||
if (r.playbook_id) priorIds.push(r.playbook_id);
|
||||
priorAnswers.push(r.answer_preview);
|
||||
log(` → iter ${i}: ${r.errors_recovered.length} errors recovered, ${r.continuation_retries} continuations, tree-split=${r.tree_split_fired}, rescue=${r.rescue_triggered}, ${r.duration_ms}ms`);
|
||||
await writeFile(`${OUT_DIR}/iter_${i}.json`, JSON.stringify(r, null, 2));
|
||||
}
|
||||
// Check whether iter 6 actually cited prior pb:IDs in its answer.
|
||||
// Playbook IDs look like `pb-seed-<hex>` so the regex needs to allow
|
||||
// hyphens + letters inside the brackets, not just hex chars.
|
||||
const iter6 = results[5];
|
||||
const citationsHonored = iter6 ? (iter6.answer_preview.match(/\[pb:[\w-]+\]/gi)?.length ?? 0) : 0;
|
||||
|
||||
// ─── Phase 3: summary ──────────────────────────────────────────
|
||||
const summary = {
|
||||
run_nonce: RUN_NONCE,
|
||||
ran_at: new Date().toISOString(),
|
||||
prd_chars: prd.length,
|
||||
prd_chunks: allChunks.length,
|
||||
iterations: results.length,
|
||||
total_cloud_calls: results.reduce((s, r) => s + r.cloud_calls_total, 0),
|
||||
total_continuation_retries: results.reduce((s, r) => s + r.continuation_retries, 0),
|
||||
total_errors_recovered: results.reduce((s, r) => s + r.errors_recovered.length, 0),
|
||||
tree_splits_fired: results.filter(r => r.tree_split_fired).length,
|
||||
rescues_triggered: results.filter(r => r.rescue_triggered).length,
|
||||
iter6_received_prior_ids: results[5]?.citations_from_prior_iterations.length ?? 0,
|
||||
iter6_actually_cited_in_answer: citationsHonored,
|
||||
iter3_task_attempts: results[2]?.task_attempts_made ?? 0,
|
||||
iter3_task_retries: results[2]?.task_retry_history.length ?? 0,
|
||||
max_task_attempts_any_iter: Math.max(...results.map(r => r.task_attempts_made)),
|
||||
total_duration_ms: results.reduce((s, r) => s + r.duration_ms, 0),
|
||||
overall: results.length === 6 && results.every(r => r.playbook_id !== null) ? "PASS" : "PARTIAL",
|
||||
};
|
||||
await writeFile(`${OUT_DIR}/summary.json`, JSON.stringify(summary, null, 2));
|
||||
|
||||
log("");
|
||||
log(`══════ SUMMARY ${summary.overall} ══════`);
|
||||
log(` 6 iterations, ${summary.total_cloud_calls} cloud calls, ${summary.total_errors_recovered} errors recovered`);
|
||||
log(` tree-splits: ${summary.tree_splits_fired}/6 continuations: ${summary.total_continuation_retries} rescues: ${summary.rescues_triggered}`);
|
||||
log(` iter 6 received ${summary.iter6_received_prior_ids} prior IDs, cited ${summary.iter6_actually_cited_in_answer} [pb:...] markers in its answer`);
|
||||
log(` iter 3 task-retry loop: ${summary.iter3_task_attempts} attempts (${summary.iter3_task_retries} prior-failure retries before success)`);
|
||||
log(` total duration: ${(summary.total_duration_ms / 1000).toFixed(1)}s`);
|
||||
log("");
|
||||
for (const r of results) {
|
||||
const flags = [
|
||||
r.tree_split_fired ? "tree-split" : "",
|
||||
r.continuation_retries > 0 ? `cont=${r.continuation_retries}` : "",
|
||||
r.rescue_triggered ? "rescued" : "",
|
||||
r.errors_recovered.length > 0 ? `err=${r.errors_recovered.length}` : "",
|
||||
].filter(Boolean).join(" ");
|
||||
log(` iter ${r.iteration}: ${r.tokens_prompt}+${r.tokens_completion} tok, ${r.duration_ms}ms ${flags ? `[${flags}]` : ""}`);
|
||||
}
|
||||
log("");
|
||||
log(`artifacts: ${OUT_DIR}/{iter_1..6.json, summary.json}`);
|
||||
process.exit(summary.overall === "PASS" ? 0 : 1);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error("[enrich] fatal:", e); process.exit(2); });
|
||||
@@ -0,0 +1,267 @@
|
||||
// Hard-task escalation test. The task is deliberately constructed so
|
||||
// that a local 7B model (qwen3.5:latest) will miss at least one of the
|
||||
// validation rules. Watch the escalation ladder:
|
||||
// 1. qwen3.5:latest (local 7B) — likely fails
|
||||
// 2. qwen3:latest (local 7B) — likely fails differently
|
||||
// 3. gpt-oss:20b (cloud 20B) — may fail
|
||||
// 4. gpt-oss:120b (cloud 120B) — should succeed
|
||||
// 5. gpt-oss:120b w/ prior-attempt errors injected — retry with context
|
||||
// 6. absolute last ditch: return best-so-far with failure annotation
|
||||
//
|
||||
// Each attempt:
|
||||
// - Calls the model via /v1/chat
|
||||
// - Validates the output against a strict rubric
|
||||
// - On fail: records the specific rubric violations + the partial
|
||||
// output, injects both into the next attempt's prompt as "here's
|
||||
// what's wrong, fix it specifically"
|
||||
// - On success: exit loop
|
||||
//
|
||||
// Run: bun run tests/real-world/hard_task_escalation.ts
|
||||
|
||||
import { writeFile, mkdir } from "node:fs/promises";
|
||||
|
||||
const GATEWAY = "http://localhost:3100";
|
||||
const OUT_DIR = `/home/profit/lakehouse/tests/real-world/runs/hard_task_${Date.now().toString(36)}`;
|
||||
const MAX_ATTEMPTS = 6;
|
||||
|
||||
// The hard task. Specific enough that a small model will miss at
|
||||
// least one rule. Not purely knowledge-based — it's a code-generation
|
||||
// task with strict structural constraints.
|
||||
const TASK = `Write a complete Rust async function with the EXACT signature:
|
||||
|
||||
pub async fn check_drift_batched(refs: Vec<DocRef>) -> Result<Vec<String>, String>
|
||||
|
||||
It must:
|
||||
1. Group refs by tool name (case-insensitive — use .to_ascii_lowercase())
|
||||
2. Issue parallel HTTP GET requests to http://localhost:3900/docs/{tool}/diff?since={snippet_hash}
|
||||
3. Use reqwest and a JoinSet/Semaphore to cap concurrent in-flight requests at 4
|
||||
4. On HTTP 5xx, retry with exponential backoff: sleep 250ms, then 500ms, then 1000ms, then give up on that tool
|
||||
5. Parse the response JSON: {"drifted": bool, ...}. Return a Vec<String> of tool names where drifted == true
|
||||
6. All errors bubble via ? or Result — NO .unwrap(), NO .expect(), NO panic!()
|
||||
7. Include rustdoc /// comments on the function and each internal helper
|
||||
|
||||
Assume this struct is already imported:
|
||||
|
||||
pub struct DocRef { pub tool: String, pub snippet_hash: Option<String>, pub version_seen: String }
|
||||
|
||||
Output ONLY the Rust code. No prose, no markdown fences, no explanation. Start directly with the /// doc comment.`;
|
||||
|
||||
// Escalation ladder — small-local → large-local → cloud → specialist
|
||||
// cloud → trillion-param cloud. Corrected 2026-04-22 per J:
|
||||
// gpt-oss:20b is LOCAL (ollama list confirms 13 GB on disk), and the
|
||||
// final escalation tier should be kimi-k2:1t (the biggest model
|
||||
// we have access to on Ollama Cloud).
|
||||
const LADDER: Array<{ provider: "ollama" | "ollama_cloud"; model: string; note: string }> = [
|
||||
{ provider: "ollama", model: "qwen3.5:latest", note: "local 7B" },
|
||||
{ provider: "ollama", model: "qwen3:latest", note: "local 7B (different) " },
|
||||
{ provider: "ollama", model: "gpt-oss:20b", note: "local 20B" }, // FIXED: local, not cloud
|
||||
{ provider: "ollama_cloud", model: "gpt-oss:120b", note: "cloud 120B" },
|
||||
{ provider: "ollama_cloud", model: "devstral-2:123b", note: "cloud 123B (coding specialist)" },
|
||||
// NOTE 2026-04-22 — J wanted Kimi as the last escalation but Kimi
|
||||
// K2.5/K2.6 both return "this model requires a subscription" on our
|
||||
// current Ollama Cloud key. mistral-large-3:675b is the biggest
|
||||
// model actually provisioned on this key (verified via direct curl
|
||||
// to ollama.com/api/generate). Upgrade path: Ollama Cloud Pro →
|
||||
// swap this line to kimi-k2.5 or kimi-k2.6:cloud.
|
||||
{ provider: "ollama_cloud", model: "mistral-large-3:675b", note: "cloud 675B (biggest available on current key; kimi-k2.x needs pro subscription)" },
|
||||
];
|
||||
|
||||
// Validation rubric — the answer must pass all of these to be accepted.
|
||||
interface RubricResult {
|
||||
passed: boolean;
|
||||
violations: string[];
|
||||
passed_rules: string[];
|
||||
}
|
||||
|
||||
function validate(code: string): RubricResult {
|
||||
const violations: string[] = [];
|
||||
const passed: string[] = [];
|
||||
|
||||
const check = (rule: string, ok: boolean) => { ok ? passed.push(rule) : violations.push(rule); };
|
||||
|
||||
check("has pub async fn check_drift_batched signature",
|
||||
/pub\s+async\s+fn\s+check_drift_batched\s*\(/.test(code));
|
||||
check("takes Vec<DocRef> argument",
|
||||
/refs\s*:\s*Vec\s*<\s*DocRef\s*>/.test(code));
|
||||
check("returns Result<Vec<String>, String>",
|
||||
/Result\s*<\s*Vec\s*<\s*String\s*>\s*,\s*String\s*>/.test(code));
|
||||
check("uses reqwest",
|
||||
/\breqwest\b/i.test(code));
|
||||
check("references JoinSet or Semaphore for concurrency",
|
||||
/\bJoinSet\b|\bSemaphore\b/i.test(code));
|
||||
check("bounds concurrency at 4",
|
||||
/\b4\b/.test(code) && (/Semaphore\s*::\s*new\s*\(\s*4\b/.test(code) || /permits\s*:\s*4\b/.test(code) || /limit\s*:\s*4\b/.test(code) || /max\s*:\s*4\b/.test(code) || /capacity\s*:\s*4\b/.test(code)));
|
||||
// Exponential backoff — models express this several ways. Accept
|
||||
// any recognizable doubling pattern starting at 250ms. 2026-04-22:
|
||||
// devstral-2:123b wrote `retry_delay *= 2` which my earlier regex
|
||||
// rejected even though the code is correct. Broadening rubric to
|
||||
// match all idiomatic doubling forms.
|
||||
const hasSeed250 = /Duration\s*::\s*from_millis\s*\(\s*250\b/.test(code)
|
||||
|| /millis\s*\(\s*250\b/.test(code);
|
||||
const hasDoublingPattern = /250\s*\*\s*2/.test(code) // 250 * 2^n literal
|
||||
|| /<<\s*\d+/.test(code) // bit-shift
|
||||
|| /\.pow\s*\(/.test(code) // 2u32.pow(attempt)
|
||||
|| /\*=\s*2\b/.test(code) // delay *= 2 ← was missing
|
||||
|| /\*\s*2\s*;/.test(code) // delay = delay * 2;
|
||||
|| /saturating_mul\s*\(\s*2\b/.test(code); // saturating doubling
|
||||
check("has 250ms backoff seed",
|
||||
hasSeed250);
|
||||
check("reaches 500ms backoff (literal or doubling from 250)",
|
||||
/Duration\s*::\s*from_millis\s*\(\s*500\b/.test(code)
|
||||
|| /millis\s*\(\s*500\b/.test(code)
|
||||
|| (hasSeed250 && hasDoublingPattern));
|
||||
check("reaches 1000ms backoff (literal or doubling to 1000)",
|
||||
/Duration\s*::\s*from_millis\s*\(\s*1000\b/.test(code)
|
||||
|| /millis\s*\(\s*1000\b/.test(code)
|
||||
|| (hasSeed250 && hasDoublingPattern));
|
||||
check("case-insensitive tool grouping (to_ascii_lowercase)",
|
||||
/to_ascii_lowercase|to_lowercase/.test(code));
|
||||
check("NO .unwrap() — all errors bubble via ?",
|
||||
!/\.unwrap\s*\(\s*\)/.test(code));
|
||||
check("NO .expect(...) — all errors bubble via ?",
|
||||
!/\.expect\s*\(/.test(code));
|
||||
check("NO panic!() / unimplemented!() / todo!()",
|
||||
!/\bpanic!\s*\(|\bunimplemented!\s*\(|\btodo!\s*\(/.test(code));
|
||||
check("has rustdoc /// comments",
|
||||
/\/\/\//.test(code));
|
||||
check("reasonable length (> 500 chars)",
|
||||
code.length > 500);
|
||||
|
||||
return { passed: violations.length === 0, violations, passed_rules: passed };
|
||||
}
|
||||
|
||||
function log(msg: string) { console.log(`[hard] ${msg}`); }
|
||||
|
||||
async function chat(opts: {
|
||||
provider: "ollama" | "ollama_cloud",
|
||||
model: string,
|
||||
prompt: string,
|
||||
}): Promise<{ content: string; error?: string }> {
|
||||
try {
|
||||
const r = await fetch(`${GATEWAY}/v1/chat`, {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
provider: opts.provider,
|
||||
model: opts.model,
|
||||
messages: [{ role: "user", content: opts.prompt }],
|
||||
max_tokens: 2500,
|
||||
temperature: 0.2,
|
||||
think: false,
|
||||
}),
|
||||
signal: AbortSignal.timeout(240000),
|
||||
});
|
||||
if (!r.ok) return { content: "", error: `/v1/chat ${r.status}: ${(await r.text()).slice(0, 300)}` };
|
||||
const j: any = await r.json();
|
||||
return { content: j.choices?.[0]?.message?.content ?? "" };
|
||||
} catch (e) {
|
||||
return { content: "", error: (e as Error).message };
|
||||
}
|
||||
}
|
||||
|
||||
interface AttemptRecord {
|
||||
n: number;
|
||||
provider: string;
|
||||
model: string;
|
||||
duration_ms: number;
|
||||
content_chars: number;
|
||||
error: string | null;
|
||||
rubric_violations: string[];
|
||||
rubric_passed: string[];
|
||||
accepted: boolean;
|
||||
}
|
||||
|
||||
function extractCode(raw: string): string {
|
||||
// Strip common fence wrappers
|
||||
const m = raw.match(/```(?:rust)?\s*\n([\s\S]*?)```/);
|
||||
if (m) return m[1].trim();
|
||||
return raw.trim();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(OUT_DIR, { recursive: true });
|
||||
log(`output: ${OUT_DIR}`);
|
||||
log(`task: ${TASK.slice(0, 120)}...`);
|
||||
log("");
|
||||
|
||||
const attempts: AttemptRecord[] = [];
|
||||
let acceptedCode: string | null = null;
|
||||
|
||||
for (let i = 0; i < MAX_ATTEMPTS; i++) {
|
||||
const n = i + 1;
|
||||
const rung = LADDER[i] ?? LADDER[LADDER.length - 1];
|
||||
|
||||
// Build the prompt: base task + prior failures' learning blocks
|
||||
let priorLearning = "";
|
||||
if (attempts.length > 0) {
|
||||
priorLearning = `\n\n═══ PRIOR ATTEMPTS FAILED. Fix these exact issues: ═══\n`;
|
||||
for (const a of attempts) {
|
||||
priorLearning += `Attempt ${a.n} (${a.provider}/${a.model}, ${a.content_chars} chars) violations:\n`;
|
||||
for (const v of a.rubric_violations) priorLearning += ` - ${v}\n`;
|
||||
if (a.error) priorLearning += ` [error: ${a.error.slice(0, 120)}]\n`;
|
||||
}
|
||||
priorLearning += `═══ end prior attempts ═══\n\nDO NOT repeat the above violations. Address each one explicitly.`;
|
||||
}
|
||||
|
||||
log(`attempt ${n}/${MAX_ATTEMPTS}: ${rung.provider}::${rung.model}${priorLearning ? " [w/ learning]" : ""}`);
|
||||
const t0 = Date.now();
|
||||
const r = await chat({ provider: rung.provider, model: rung.model, prompt: TASK + priorLearning });
|
||||
const dur = Date.now() - t0;
|
||||
|
||||
const code = extractCode(r.content);
|
||||
const rubric = code ? validate(code) : { passed: false, violations: ["empty response"], passed_rules: [] };
|
||||
|
||||
const record: AttemptRecord = {
|
||||
n,
|
||||
provider: rung.provider,
|
||||
model: rung.model,
|
||||
duration_ms: dur,
|
||||
content_chars: code.length,
|
||||
error: r.error ?? null,
|
||||
rubric_violations: rubric.violations,
|
||||
rubric_passed: rubric.passed_rules,
|
||||
accepted: rubric.passed,
|
||||
};
|
||||
attempts.push(record);
|
||||
|
||||
log(` → ${dur}ms, ${code.length} chars, ${rubric.passed_rules.length} rules passed / ${rubric.violations.length} failed${r.error ? `, err: ${r.error.slice(0, 80)}` : ""}`);
|
||||
for (const v of rubric.violations.slice(0, 5)) log(` ✗ ${v}`);
|
||||
|
||||
await writeFile(`${OUT_DIR}/attempt_${n}.txt`, code);
|
||||
await writeFile(`${OUT_DIR}/attempt_${n}.json`, JSON.stringify(record, null, 2));
|
||||
|
||||
if (rubric.passed) {
|
||||
log(` ✅ ACCEPTED on attempt ${n}`);
|
||||
acceptedCode = code;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {
|
||||
task: TASK.slice(0, 200),
|
||||
total_attempts: attempts.length,
|
||||
accepted: acceptedCode !== null,
|
||||
accepted_on_attempt: acceptedCode ? attempts.findIndex(a => a.accepted) + 1 : null,
|
||||
escalation_path: attempts.map(a => `${a.provider}/${a.model}`),
|
||||
per_attempt_pass_counts: attempts.map(a => a.rubric_passed.length),
|
||||
per_attempt_violation_counts: attempts.map(a => a.rubric_violations.length),
|
||||
total_duration_ms: attempts.reduce((s, a) => s + a.duration_ms, 0),
|
||||
};
|
||||
await writeFile(`${OUT_DIR}/summary.json`, JSON.stringify(summary, null, 2));
|
||||
|
||||
log("");
|
||||
log(`═══ RESULT ═══`);
|
||||
log(`attempts: ${summary.total_attempts}`);
|
||||
log(`accepted: ${summary.accepted} ${summary.accepted ? `on attempt ${summary.accepted_on_attempt}` : ""}`);
|
||||
log(`escalation path:`);
|
||||
for (const [i, a] of attempts.entries()) {
|
||||
const mark = a.accepted ? "✅" : "❌";
|
||||
log(` ${mark} attempt ${i + 1}: ${a.provider}/${a.model} — ${a.rubric_passed.length}/${a.rubric_passed.length + a.rubric_violations.length} rules passed, ${a.duration_ms}ms`);
|
||||
}
|
||||
log("");
|
||||
log(`total time: ${(summary.total_duration_ms / 1000).toFixed(1)}s`);
|
||||
log(`artifacts: ${OUT_DIR}/{attempt_1..N.{txt,json}, summary.json}`);
|
||||
|
||||
process.exit(summary.accepted ? 0 : 1);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error("[hard] fatal:", e); process.exit(2); });
|
||||
@@ -0,0 +1,186 @@
|
||||
// Nine-consecutive audit runner — empirical test of the predictive-
|
||||
// compounding property. Runs the audit pipeline 9 times against the
|
||||
// same PR (each time with a new diff from Gitea), captures the
|
||||
// verdict + audit_lessons state after each run, and reports whether
|
||||
// the KB stabilizes or drifts.
|
||||
//
|
||||
// What we expect (favorable compounding):
|
||||
// - signature_count grows sublinearly (same patterns recur, so
|
||||
// distinct-signature count stabilizes fast)
|
||||
// - verdict settles on a stable value after run 2-3 (first audit
|
||||
// establishes baseline, rest repeat)
|
||||
// - confidence stays LOW for all signatures (same PR repeatedly)
|
||||
// - NO new recurring findings fire because confidence < 0.3 on
|
||||
// same-PR noise (kb_index rating policy)
|
||||
//
|
||||
// What would indicate drift (the thing we want to prove DOESN'T happen):
|
||||
// - signature_count grows linearly — each run produces new signatures
|
||||
// - verdict oscillates (block → approve → block ...)
|
||||
// - confidence inflates — kb_index rating escalates on repeated runs
|
||||
//
|
||||
// Run: bun run tests/real-world/nine_consecutive_audits.ts
|
||||
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { aggregate } from "../../auditor/kb_index.ts";
|
||||
import { getPrSnapshot } from "../../auditor/gitea.ts";
|
||||
import { auditPr } from "../../auditor/audit.ts";
|
||||
|
||||
const REPO = "/home/profit/lakehouse";
|
||||
const AUDIT_LESSONS = `${REPO}/data/_kb/audit_lessons.jsonl`;
|
||||
const VERDICTS_DIR = `${REPO}/data/_auditor/verdicts`;
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
const RUNS = Number(process.env.LH_AUDIT_RUNS ?? 9);
|
||||
const TARGET_PR = Number(process.env.LH_AUDIT_PR ?? 8);
|
||||
const SKIP_INFERENCE = process.env.LH_AUDITOR_SKIP_INFERENCE !== "0";
|
||||
const RESET_KB = process.env.LH_RESET_KB === "1";
|
||||
|
||||
async function waitForVerdict(prNum: number, sha: string, deadlineMs: number): Promise<any> {
|
||||
const short = sha.slice(0, 12);
|
||||
const path = join(VERDICTS_DIR, `${prNum}-${short}.json`);
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < deadlineMs) {
|
||||
try {
|
||||
const raw = await readFile(path, "utf8");
|
||||
return JSON.parse(raw);
|
||||
} catch { /* not yet */ }
|
||||
await Bun.sleep(POLL_INTERVAL_MS);
|
||||
}
|
||||
throw new Error(`no verdict file after ${deadlineMs}ms: ${path}`);
|
||||
}
|
||||
|
||||
async function captureAggState(): Promise<{ sig_count: number; max_count: number; max_confidence: number; top3: Array<{ sig: string; count: number; conf: number; summary: string }> }> {
|
||||
const agg = await aggregate<any>(AUDIT_LESSONS, {
|
||||
keyFn: (r) => r?.signature,
|
||||
scopeFn: (r) => (r?.pr_number !== undefined ? `pr-${r.pr_number}` : undefined),
|
||||
});
|
||||
const list = Array.from(agg.values()).sort((a, b) => b.count - a.count);
|
||||
const recurring = list.filter(r => r.count >= 2);
|
||||
const recurringMaxCount = recurring.length > 0 ? Math.max(...recurring.map(a => a.count)) : 0;
|
||||
const recurringMaxConf = recurring.length > 0 ? Math.max(...recurring.map(a => a.confidence)) : 0;
|
||||
return {
|
||||
sig_count: list.length,
|
||||
max_count: list[0]?.count ?? 0,
|
||||
max_confidence: recurringMaxConf,
|
||||
recurring_max_count: recurringMaxCount,
|
||||
top3: list.slice(0, 3).map(a => ({
|
||||
sig: a.signature,
|
||||
count: a.count,
|
||||
conf: a.confidence,
|
||||
summary: a.representative_summary.slice(0, 80),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
interface RunRecord {
|
||||
run: number;
|
||||
sha: string;
|
||||
verdict_overall: string;
|
||||
findings_total: number;
|
||||
findings_block: number;
|
||||
findings_warn: number;
|
||||
findings_info: number;
|
||||
audit_duration_ms: number;
|
||||
claims_total: number;
|
||||
claims_empirical: number;
|
||||
kb_sig_count_after: number;
|
||||
kb_max_count_after: number;
|
||||
kb_max_confidence_after: number;
|
||||
kb_recurring_max_count: number;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`[nine] target PR: #${TARGET_PR}`);
|
||||
console.log(`[nine] runs: ${RUNS}`);
|
||||
console.log(`[nine] skip_inference: ${SKIP_INFERENCE}`);
|
||||
console.log(`[nine] reset_kb: ${RESET_KB}`);
|
||||
console.log(`[nine] audit_lessons.jsonl: ${AUDIT_LESSONS}`);
|
||||
|
||||
if (RESET_KB) {
|
||||
console.log("[nine] clearing audit_lessons.jsonl for clean test...");
|
||||
await writeFile(AUDIT_LESSONS, "");
|
||||
}
|
||||
console.log("");
|
||||
|
||||
const pr = await getPrSnapshot(TARGET_PR);
|
||||
console.log(`[nine] PR #${pr.number}: "${pr.title}" (head=${pr.head_sha.slice(0, 12)})`);
|
||||
console.log(`[nine] files in diff: ${pr.files.length}`);
|
||||
console.log("");
|
||||
|
||||
const baseline = await captureAggState();
|
||||
console.log(`[nine] baseline: sig_count=${baseline.sig_count} max_count=${baseline.max_count} max_conf=${baseline.max_confidence.toFixed(2)}`);
|
||||
console.log("");
|
||||
|
||||
const records: RunRecord[] = [];
|
||||
for (let n = 1; n <= RUNS; n++) {
|
||||
const t0 = Date.now();
|
||||
console.log(`─── run ${n}/${RUNS} ───`);
|
||||
|
||||
const verdict = await auditPr(pr, {
|
||||
dry_run: true,
|
||||
skip_dynamic: true,
|
||||
skip_inference: SKIP_INFERENCE,
|
||||
});
|
||||
|
||||
console.log(` sha ${verdict.head_sha.slice(0, 12)}`);
|
||||
const after = await captureAggState();
|
||||
const rec: RunRecord = {
|
||||
run: n,
|
||||
sha: verdict.head_sha.slice(0, 12),
|
||||
verdict_overall: String(verdict.overall),
|
||||
findings_total: Number(verdict.metrics?.findings_total ?? 0),
|
||||
findings_block: Number(verdict.metrics?.findings_block ?? 0),
|
||||
findings_warn: Number(verdict.metrics?.findings_warn ?? 0),
|
||||
findings_info: Number(verdict.metrics?.findings_info ?? 0),
|
||||
audit_duration_ms: Number(verdict.metrics?.audit_duration_ms ?? 0),
|
||||
claims_total: Number(verdict.metrics?.claims_total ?? 0),
|
||||
claims_empirical: Number(verdict.metrics?.claims_empirical ?? 0),
|
||||
kb_sig_count_after: after.sig_count,
|
||||
kb_max_count_after: after.max_count,
|
||||
kb_max_confidence_after: after.max_confidence,
|
||||
kb_recurring_max_count: after.recurring_max_count,
|
||||
};
|
||||
records.push(rec);
|
||||
console.log(` verdict=${rec.verdict_overall} findings=${rec.findings_total} (b=${rec.findings_block} w=${rec.findings_warn})`);
|
||||
console.log(` kb after: sig=${rec.kb_sig_count_after} max_count=${rec.kb_max_count_after} recurring_max=${rec.kb_recurring_max_count} max_conf=${rec.kb_max_confidence_after.toFixed(2)}`);
|
||||
console.log(` elapsed: ${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
console.log("═══ FINAL ═══");
|
||||
console.log("run | verdict | find | block warn info | dur_s | kb_sig max_count max_conf");
|
||||
for (const r of records) {
|
||||
console.log(
|
||||
` ${String(r.run).padStart(1)} | ${r.verdict_overall.padEnd(16)} | ${String(r.findings_total).padStart(4)} | ${String(r.findings_block).padStart(5)} ${String(r.findings_warn).padStart(5)} ${String(r.findings_info).padStart(5)} | ${(r.audit_duration_ms / 1000).toFixed(1).padStart(5)} | ${String(r.kb_sig_count_after).padStart(6)} ${String(r.kb_max_count_after).padStart(9)} ${r.kb_max_confidence_after.toFixed(2)}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("═══ COMPOUNDING PROPERTY ═══");
|
||||
const sigDelta = records[records.length - 1].kb_sig_count_after - baseline.sig_count;
|
||||
const maxConf = records[records.length - 1].kb_max_confidence_after;
|
||||
const recurringMax = records[records.length - 1].kb_recurring_max_count;
|
||||
console.log(` signatures added over ${RUNS} runs: ${sigDelta}`);
|
||||
console.log(` max recurring count after run ${RUNS}: ${recurringMax} (same-PR recurrences per signature)`);
|
||||
console.log(` max confidence after run ${RUNS}: ${maxConf.toFixed(2)} (expect LOW — same-PR should not inflate)`);
|
||||
|
||||
const verdictSet = new Set(records.map(r => r.verdict_overall));
|
||||
if (verdictSet.size === 1) {
|
||||
console.log(` verdict stable: all ${RUNS} runs returned '${[...verdictSet][0]}' ✓`);
|
||||
} else {
|
||||
console.log(` verdict oscillated across runs: ${[...verdictSet].join(" | ")} ✗`);
|
||||
}
|
||||
|
||||
if (maxConf < 0.6 && recurringMax < 5) {
|
||||
console.log(` confidence policy holding: same-PR noise stays below escalation threshold ✓`);
|
||||
} else {
|
||||
console.log(` ⚠ cross-cutting pattern detected (conf=${maxConf.toFixed(2)}, recurring=${recurringMax}) — kb_index policy escalated`);
|
||||
}
|
||||
|
||||
const jsonOut = `${REPO}/tests/real-world/runs/nine_consecutive_${Date.now().toString(36)}.json`;
|
||||
await Bun.write(jsonOut, JSON.stringify({ target_pr: TARGET_PR, baseline, records }, null, 2));
|
||||
console.log("");
|
||||
console.log(` report: ${jsonOut}`);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error("[nine] fatal:", e); process.exit(1); });
|
||||
@@ -0,0 +1,487 @@
|
||||
// scrum_applier.ts — the auto-apply pipeline.
|
||||
//
|
||||
// Turns the scrum master's signal into real commits. Reads
|
||||
// data/_kb/scrum_reviews.jsonl, filters to rows where the scrum's
|
||||
// own confidence is high enough to trust auto-apply (gradient_tier
|
||||
// auto OR confidence_avg ≥ 90), asks a patch-emitting model to
|
||||
// produce concrete old_string/new_string pairs, applies them via
|
||||
// text replacement, runs `cargo check` after each, commits on green
|
||||
// and reverts on red.
|
||||
//
|
||||
// Runs on its own branch (never on main). Every action is recorded
|
||||
// in data/_kb/auto_apply.jsonl so the auditor and future iterations
|
||||
// can see what landed and what reverted.
|
||||
//
|
||||
// Usage:
|
||||
// bun run tests/real-world/scrum_applier.ts # dry-run, print only
|
||||
// LH_APPLIER_COMMIT=1 bun run tests/real-world/scrum_applier.ts # actually apply
|
||||
//
|
||||
// Env:
|
||||
// LH_APPLIER_BRANCH — branch name (default: "scrum/auto-apply-${Date.now()}")
|
||||
// LH_APPLIER_MIN_CONF — minimum confidence_avg, default 90
|
||||
// LH_APPLIER_MAX_FILES — cap on files per run (default 5, keeps diffs reviewable)
|
||||
// LH_APPLIER_COMMIT — "1" to actually commit; otherwise dry-run
|
||||
// LH_APPLIER_MODEL — patch-emitting model (default: kimi-k2:1t)
|
||||
|
||||
import { readFile, writeFile, appendFile } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const REPO = "/home/profit/lakehouse";
|
||||
const GATEWAY = "http://localhost:3100";
|
||||
const SCRUM_REVIEWS = `${REPO}/data/_kb/scrum_reviews.jsonl`;
|
||||
const AUDIT_LOG = `${REPO}/data/_kb/auto_apply.jsonl`;
|
||||
|
||||
const MIN_CONF = Number(process.env.LH_APPLIER_MIN_CONF ?? 90);
|
||||
const MAX_FILES = Number(process.env.LH_APPLIER_MAX_FILES ?? 5);
|
||||
const COMMIT = process.env.LH_APPLIER_COMMIT === "1";
|
||||
// LH_APPLIER_FILES — comma-separated repo-relative paths. When set,
|
||||
// constrains eligible reviews to ONLY those files. Used by the autonomous
|
||||
// loop so the applier patches what scrum just reviewed in this iter,
|
||||
// instead of pulling the highest-confidence file from global review history.
|
||||
const TARGET_FILES = (process.env.LH_APPLIER_FILES ?? "")
|
||||
.split(",").map(s => s.trim()).filter(Boolean);
|
||||
// Default patch-emitter model — qwen3-coder:480b is the coding specialist
|
||||
// in the scrum ladder (rung 2). Swapped in from kimi-k2:1t after 2026-04-24
|
||||
// data showed kimi-k2:1t produces architectural patches that cascade across
|
||||
// the file (rename a field → 20 broken call sites). qwen3-coder is tuned
|
||||
// for targeted code changes and tends to stay within the mechanical-patch
|
||||
// constraint the prompt asks for. LLM Team's /api/run?mode=patch would be
|
||||
// the ideal choice but that mode isn't registered in llm_team_ui.py yet.
|
||||
// Default patch emitter swapped to OpenRouter Grok 4.1 fast (2026-04-25)
|
||||
// after observing the prior default (ollama_cloud::qwen3-coder:480b) sit
|
||||
// at 429 throttle and never produce patches. Grok 4.1 fast: $0.20/$0.50
|
||||
// per M, 2M ctx, proven to emit precise structured patches in observer
|
||||
// hand-review tests. Override with LH_APPLIER_MODEL + LH_APPLIER_PROVIDER.
|
||||
const MODEL = process.env.LH_APPLIER_MODEL ?? "x-ai/grok-4.1-fast";
|
||||
const PROVIDER = (process.env.LH_APPLIER_PROVIDER ?? "openrouter") as "ollama_cloud" | "openrouter" | "ollama";
|
||||
const BRANCH = process.env.LH_APPLIER_BRANCH ?? `scrum/auto-apply-${Date.now().toString(36)}`;
|
||||
|
||||
// Deny-list — anything whose path starts with one of these is skipped
|
||||
// regardless of how confident the scrum is. Config / systemd / docs /
|
||||
// auditor itself are off limits for auto-apply; they need a human.
|
||||
const DENY_PREFIXES = [
|
||||
"config/",
|
||||
"ops/",
|
||||
"auditor/",
|
||||
"docs/",
|
||||
"data/",
|
||||
"/etc/",
|
||||
"mcp-server/",
|
||||
"ui/",
|
||||
"sidecar/",
|
||||
"scripts/",
|
||||
];
|
||||
|
||||
function log(msg: string) { console.log(`[applier] ${msg}`); }
|
||||
|
||||
async function sh(cmd: string[], cwd = REPO): Promise<{ stdout: string; stderr: string; code: number }> {
|
||||
return new Promise((resolve) => {
|
||||
const p = spawn(cmd[0], cmd.slice(1), { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = ""; let err = "";
|
||||
p.stdout.on("data", (d) => { out += d.toString(); });
|
||||
p.stderr.on("data", (d) => { err += d.toString(); });
|
||||
p.on("close", (code) => resolve({ stdout: out, stderr: err, code: code ?? 1 }));
|
||||
});
|
||||
}
|
||||
|
||||
async function auditLog(row: Record<string, any>) {
|
||||
const line = JSON.stringify({ ...row, ts: new Date().toISOString() }) + "\n";
|
||||
await appendFile(AUDIT_LOG, line);
|
||||
}
|
||||
|
||||
async function chat(opts: {
|
||||
provider: "ollama_cloud" | "openrouter" | "ollama";
|
||||
model: string;
|
||||
prompt: string;
|
||||
max_tokens?: number;
|
||||
}): Promise<{ content: string; error?: string }> {
|
||||
try {
|
||||
const r = await fetch(`${GATEWAY}/v1/chat`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
provider: opts.provider,
|
||||
model: opts.model,
|
||||
messages: [{ role: "user", content: opts.prompt }],
|
||||
max_tokens: opts.max_tokens ?? 1500,
|
||||
temperature: 0.1,
|
||||
}),
|
||||
signal: AbortSignal.timeout(180000),
|
||||
});
|
||||
if (!r.ok) return { content: "", error: `${r.status}: ${(await r.text()).slice(0, 300)}` };
|
||||
const j: any = await r.json();
|
||||
return { content: j.choices?.[0]?.message?.content ?? "" };
|
||||
} catch (e) {
|
||||
return { content: "", error: String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
interface ScrumReview {
|
||||
file: string;
|
||||
reviewed_at: string;
|
||||
accepted_model: string;
|
||||
suggestions_preview: string;
|
||||
confidences_per_finding?: number[];
|
||||
confidence_avg?: number | null;
|
||||
confidence_min?: number | null;
|
||||
gradient_tier?: string;
|
||||
gradient_tier_avg?: string;
|
||||
verdict?: string;
|
||||
critical_failures_count?: number;
|
||||
schema_version?: number;
|
||||
}
|
||||
|
||||
async function loadLatestReviews(): Promise<Map<string, ScrumReview>> {
|
||||
// Map of file → latest review for that file. Ordered by reviewed_at.
|
||||
if (!existsSync(SCRUM_REVIEWS)) return new Map();
|
||||
const text = await readFile(SCRUM_REVIEWS, "utf8");
|
||||
const rows: ScrumReview[] = text.split("\n").filter(Boolean).map(l => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter((r): r is ScrumReview => r !== null);
|
||||
// Keep the LATEST review per file.
|
||||
const latest = new Map<string, ScrumReview>();
|
||||
for (const r of rows) {
|
||||
if (!r.file) continue;
|
||||
const prev = latest.get(r.file);
|
||||
if (!prev || (r.reviewed_at > prev.reviewed_at)) latest.set(r.file, r);
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
|
||||
function passesConfidenceGate(r: ScrumReview): boolean {
|
||||
const avg = r.confidence_avg ?? 0;
|
||||
const min = r.confidence_min ?? 0;
|
||||
// Must be auto or dry_run tier AND confidence_min ≥ MIN_CONF.
|
||||
// min is the conservative tier-lower-bound (one weak finding drags
|
||||
// the whole file to "simulation" or "block" tier).
|
||||
if (r.gradient_tier === "block" || r.gradient_tier === "simulation") return false;
|
||||
return avg >= MIN_CONF && min >= 70;
|
||||
}
|
||||
|
||||
function passesDenyList(file: string): boolean {
|
||||
return !DENY_PREFIXES.some((p) => file.startsWith(p) || file === p.replace(/\/$/, ""));
|
||||
}
|
||||
|
||||
interface Patch {
|
||||
file: string;
|
||||
old_string: string;
|
||||
new_string: string;
|
||||
rationale: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
async function requestPatches(file: string, source: string, review: string): Promise<Patch[]> {
|
||||
const prompt = `You previously produced this review of ${file}:
|
||||
|
||||
─── REVIEW ───
|
||||
${review}
|
||||
─── END REVIEW ───
|
||||
|
||||
The review is high-confidence and the file is eligible for auto-apply. Produce CONCRETE PATCHES as JSON so they can be applied via string replacement.
|
||||
|
||||
HARD CONSTRAINTS (violations → patch rejected):
|
||||
1. Output ONE JSON object with a "patches" array. NO prose, no markdown fences.
|
||||
2. Each patch is {"old_string": "...", "new_string": "...", "rationale": "short", "confidence": 0-100}.
|
||||
3. "old_string" MUST appear EXACTLY ONCE in the file (verbatim, including whitespace + trailing newlines).
|
||||
4. Max diff size: 6 lines changed per patch (NOT 20). If the change needs >6 lines, emit {"patches": []} — too risky for auto-apply.
|
||||
5. Mechanical-only (the list — nothing else):
|
||||
(a) remove a #[allow(dead_code)] marker on a function now wired elsewhere
|
||||
(b) add a missing 'use' import statement
|
||||
(c) add a single field to a struct (no renames)
|
||||
(d) flip a single boolean/match-arm that doesn't cascade
|
||||
(e) add a fire-and-forget log or tracing call
|
||||
6. FORBIDDEN (automatic reject):
|
||||
• struct field RENAMES (break all call sites)
|
||||
• function signature changes
|
||||
• new modules, new traits, new dependencies
|
||||
• any change that requires editing another file to keep it compiling
|
||||
• adding a 'use' import whose symbol is NOT referenced in the same patch
|
||||
or already referenced in the surrounding file (no "import and hope")
|
||||
• rationale that describes functionality not present in the diff
|
||||
(if rationale says "destructive SQL filter", the diff must contain SQL/filter tokens)
|
||||
7. Each "new_string" MUST compile in isolation with the same surrounding code.
|
||||
8. Max 2 patches per file — quality over quantity.
|
||||
9. The "rationale" field MUST use vocabulary that appears in the new_string.
|
||||
Generic rationales like "improve code" or "add missing feature" are rejected.
|
||||
10. If you cannot produce at least one high-confidence mechanical patch under these constraints, output {"patches": []}. Don't guess.
|
||||
|
||||
─── SOURCE (${source.length} bytes) ───
|
||||
${source.slice(0, 14000)}
|
||||
─── END SOURCE ───
|
||||
|
||||
Emit ONLY the JSON object.`;
|
||||
|
||||
const r = await chat({ provider: PROVIDER, model: MODEL, prompt, max_tokens: 2500 });
|
||||
if (r.error || !r.content) return [];
|
||||
|
||||
// Strip markdown fences if model wrapped the JSON.
|
||||
let raw = r.content.trim();
|
||||
const fenceStart = raw.match(/^```(?:json)?\s*/);
|
||||
if (fenceStart) raw = raw.slice(fenceStart[0].length);
|
||||
if (raw.endsWith("```")) raw = raw.slice(0, -3).trim();
|
||||
// Find first { and last } to extract JSON block if there's prose.
|
||||
const first = raw.indexOf("{");
|
||||
const last = raw.lastIndexOf("}");
|
||||
if (first >= 0 && last > first) raw = raw.slice(first, last + 1);
|
||||
|
||||
try {
|
||||
const obj = JSON.parse(raw);
|
||||
const patches: Patch[] = (obj.patches ?? []).filter((p: any) =>
|
||||
typeof p?.old_string === "string" &&
|
||||
typeof p?.new_string === "string" &&
|
||||
p.old_string !== p.new_string &&
|
||||
p.old_string.length > 0 &&
|
||||
typeof p?.confidence === "number"
|
||||
).map((p: any) => ({
|
||||
file,
|
||||
old_string: p.old_string,
|
||||
new_string: p.new_string,
|
||||
rationale: String(p.rationale ?? ""),
|
||||
confidence: p.confidence,
|
||||
}));
|
||||
return patches;
|
||||
} catch (e) {
|
||||
log(` ${file}: patch JSON parse failed — ${String(e).slice(0, 100)}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function applyPatches(file: string, patches: Patch[]): Promise<{ applied: number; rejected: Array<{patch: Patch; reason: string}> }> {
|
||||
const full = `${REPO}/${file}`;
|
||||
let source = await readFile(full, "utf8");
|
||||
const rejected: Array<{patch: Patch; reason: string}> = [];
|
||||
let applied = 0;
|
||||
for (const p of patches) {
|
||||
// Confidence gate at the individual-patch level.
|
||||
if (p.confidence < MIN_CONF) { rejected.push({patch: p, reason: `confidence ${p.confidence} < ${MIN_CONF}`}); continue; }
|
||||
// Uniqueness gate.
|
||||
const occurrences = source.split(p.old_string).length - 1;
|
||||
if (occurrences === 0) { rejected.push({patch: p, reason: "old_string not found"}); continue; }
|
||||
if (occurrences > 1) { rejected.push({patch: p, reason: `old_string appears ${occurrences}× (not unique)`}); continue; }
|
||||
// Size gate — no patch touches > 6 lines (diff discipline; matches prompt).
|
||||
// Raised from 20 to 6 after 2026-04-24 data showed most 10-20 line patches
|
||||
// cascaded and broke the build. Mechanical changes genuinely fit in 6 lines.
|
||||
const oldLines = p.old_string.split("\n").length;
|
||||
const newLines = p.new_string.split("\n").length;
|
||||
if (Math.max(oldLines, newLines) > 6) { rejected.push({patch: p, reason: `patch too large (${Math.max(oldLines,newLines)} lines, max 6)`}); continue; }
|
||||
source = source.replace(p.old_string, p.new_string);
|
||||
applied++;
|
||||
}
|
||||
if (applied > 0) await writeFile(full, source);
|
||||
return { applied, rejected };
|
||||
}
|
||||
|
||||
async function cargoCheck(): Promise<{ green: boolean; warnings: number }> {
|
||||
const r = await sh(["cargo", "check", "--workspace"]);
|
||||
// Count warnings: cargo emits "warning:" lines on stderr (and sometimes stdout).
|
||||
// We count unique "warning: <msg>" opening lines, not continuation context.
|
||||
const combined = r.stdout + r.stderr;
|
||||
const warnings = (combined.match(/^warning: /gm) || []).length;
|
||||
return { green: r.code === 0, warnings };
|
||||
}
|
||||
|
||||
async function gitCommit(file: string, patches: Patch[]): Promise<boolean> {
|
||||
if (!COMMIT) { log(` (dry-run) would commit ${file}`); return true; }
|
||||
const addR = await sh(["git", "add", file]);
|
||||
if (addR.code !== 0) { log(` git add failed: ${addR.stderr.slice(0, 200)}`); return false; }
|
||||
const msg = `auto-apply: ${patches.length} high-confidence fix${patches.length === 1 ? "" : "es"} in ${file}\n\n${patches.map(p => `- ${p.rationale} (conf ${p.confidence}%)`).join("\n")}\n\n🤖 scrum_applier.ts`;
|
||||
const commitR = await sh(["git", "commit", "-m", msg]);
|
||||
if (commitR.code !== 0) { log(` git commit failed: ${commitR.stderr.slice(0, 200)}`); return false; }
|
||||
log(` ✓ committed ${file}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function revertFile(file: string): Promise<void> {
|
||||
await sh(["git", "checkout", "--", file]);
|
||||
}
|
||||
|
||||
// Cheap, conservative check that the rationale actually describes the
|
||||
// diff. Tokenizes the new_string (what's actually being added), drops
|
||||
// Rust keywords + common filler, and requires the rationale to share
|
||||
// at least one meaningful token. Catches rationale-diff divergence like
|
||||
// "Add destructive SQL filter" for a diff that only adds `use tracing;`.
|
||||
const STOPWORDS = new Set([
|
||||
"use", "let", "pub", "mut", "fn", "self", "the", "a", "and", "or", "in",
|
||||
"to", "for", "of", "on", "at", "add", "added", "adds", "new", "is",
|
||||
"as", "if", "else", "match", "return", "crate", "mod", "struct",
|
||||
"trait", "impl", "this", "that", "with", "from", "into", "by", "be",
|
||||
"it", "its", "not", "one", "two", "line", "lines",
|
||||
]);
|
||||
function tokenize(s: string): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const raw of s.toLowerCase().split(/[^a-z0-9_]+/)) {
|
||||
if (raw.length >= 4 && !STOPWORDS.has(raw)) out.add(raw);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
function rationaleMatchesDiff(p: Patch): boolean {
|
||||
// Only check tokens introduced by the patch (what's in new but not old).
|
||||
const newToks = tokenize(p.new_string);
|
||||
const oldToks = tokenize(p.old_string);
|
||||
const added = new Set([...newToks].filter((t) => !oldToks.has(t)));
|
||||
if (added.size === 0) return true; // pure deletion — rationale unconstrained
|
||||
const rationaleToks = tokenize(p.rationale);
|
||||
for (const t of added) if (rationaleToks.has(t)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log(`starting · min_conf=${MIN_CONF} max_files=${MAX_FILES} model=${MODEL} commit=${COMMIT}`);
|
||||
|
||||
if (COMMIT) {
|
||||
const headR = await sh(["git", "rev-parse", "--abbrev-ref", "HEAD"]);
|
||||
const currentBranch = headR.stdout.trim();
|
||||
if (currentBranch === "main") {
|
||||
log(`refusing to run on main — create a branch first or set LH_APPLIER_BRANCH`);
|
||||
const coR = await sh(["git", "checkout", "-b", BRANCH]);
|
||||
if (coR.code !== 0) { log(`could not create branch ${BRANCH}: ${coR.stderr.slice(0, 200)}`); process.exit(1); }
|
||||
log(`working branch: ${BRANCH}`);
|
||||
} else {
|
||||
log(`working branch: ${currentBranch}`);
|
||||
}
|
||||
}
|
||||
|
||||
const reviews = await loadLatestReviews();
|
||||
log(`loaded ${reviews.size} latest reviews`);
|
||||
|
||||
const eligible = [...reviews.values()].filter(r =>
|
||||
passesConfidenceGate(r) && passesDenyList(r.file) &&
|
||||
(TARGET_FILES.length === 0 || TARGET_FILES.includes(r.file))
|
||||
).sort((a, b) => (b.confidence_avg ?? 0) - (a.confidence_avg ?? 0));
|
||||
|
||||
if (TARGET_FILES.length > 0) {
|
||||
log(`LH_APPLIER_FILES set — constrained to ${TARGET_FILES.length} target file(s): ${TARGET_FILES.join(", ")}`);
|
||||
}
|
||||
log(`${eligible.length} pass confidence gate + deny-list${TARGET_FILES.length > 0 ? " + target filter" : ""}`);
|
||||
log(`taking top ${Math.min(MAX_FILES, eligible.length)} by confidence`);
|
||||
|
||||
// Establish pre-run warning baseline so post-patch cargo check can
|
||||
// reject patches that keep the build green but add new warnings.
|
||||
// This gate exists because 96b46cd landed unused imports through the
|
||||
// green-only gate — compiled fine, left a permanent dead-code warning.
|
||||
log(`baseline cargo check (warning count)...`);
|
||||
let baselineWarnings = (await cargoCheck()).warnings;
|
||||
log(`baseline warnings = ${baselineWarnings}`);
|
||||
|
||||
let committedFiles = 0;
|
||||
let revertedFiles = 0;
|
||||
let warningReverts = 0;
|
||||
let rationaleReverts = 0;
|
||||
|
||||
for (const r of eligible.slice(0, MAX_FILES)) {
|
||||
log(`${r.file} (conf_avg=${r.confidence_avg} tier=${r.gradient_tier})`);
|
||||
const full = `${REPO}/${r.file}`;
|
||||
if (!existsSync(full)) { log(` skip — file not found on disk`); continue; }
|
||||
|
||||
const source = await readFile(full, "utf8");
|
||||
const patches = await requestPatches(r.file, source, r.suggestions_preview ?? "");
|
||||
|
||||
if (patches.length === 0) {
|
||||
log(` no patches produced`);
|
||||
await auditLog({ action: "no_patches", file: r.file, reviewer_model: r.accepted_model });
|
||||
continue;
|
||||
}
|
||||
|
||||
log(` ${patches.length} candidate patches`);
|
||||
const { applied, rejected } = await applyPatches(r.file, patches);
|
||||
log(` applied ${applied}, rejected ${rejected.length}`);
|
||||
for (const rj of rejected) log(` ✗ ${rj.reason}`);
|
||||
|
||||
if (applied === 0) {
|
||||
await auditLog({ action: "all_rejected", file: r.file, rejected: rejected.map(x => x.reason) });
|
||||
continue;
|
||||
}
|
||||
|
||||
log(` running cargo check...`);
|
||||
const { green, warnings } = await cargoCheck();
|
||||
if (!green) {
|
||||
log(` ✗ build red — reverting ${r.file}`);
|
||||
await revertFile(r.file);
|
||||
revertedFiles++;
|
||||
await auditLog({ action: "build_red_reverted", file: r.file, patches_applied: applied });
|
||||
continue;
|
||||
}
|
||||
|
||||
// New-warning gate. If a patch keeps the build green but adds a
|
||||
// warning (unused import, dead_code after removed allow, etc), treat
|
||||
// it the same as build red: revert. The cargo-green gate alone let
|
||||
// 96b46cd land unused imports; this closes that door.
|
||||
if (warnings > baselineWarnings) {
|
||||
log(` ✗ warnings ${baselineWarnings} → ${warnings} (+${warnings - baselineWarnings}) — reverting ${r.file}`);
|
||||
await revertFile(r.file);
|
||||
warningReverts++;
|
||||
await auditLog({
|
||||
action: "warnings_increased_reverted",
|
||||
file: r.file,
|
||||
patches_applied: applied,
|
||||
warnings_before: baselineWarnings,
|
||||
warnings_after: warnings,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Rationale-diff alignment heuristic. At least one non-stopword
|
||||
// token from the applied patches' combined new_string must appear
|
||||
// in the rationale, OR the rationale must name the file's module.
|
||||
// This catches cases like "Add destructive SQL filter" rationale
|
||||
// with a diff that only adds `use tracing`.
|
||||
const applied_patches = patches.slice(0, applied);
|
||||
const rationale_ok = applied_patches.some((p) => rationaleMatchesDiff(p));
|
||||
if (!rationale_ok) {
|
||||
log(` ✗ rationale doesn't name anything in diff — reverting ${r.file}`);
|
||||
await revertFile(r.file);
|
||||
rationaleReverts++;
|
||||
await auditLog({
|
||||
action: "rationale_mismatch_reverted",
|
||||
file: r.file,
|
||||
patches_applied: applied,
|
||||
rationales: applied_patches.map((p) => p.rationale),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
log(` ✓ build green (warnings ${warnings}/${baselineWarnings})`);
|
||||
|
||||
// Dry-run MUST revert the file after a successful check so the
|
||||
// workspace doesn't accumulate unpushed changes across dry-runs.
|
||||
// Previously this left modified files, silently polluting state.
|
||||
if (!COMMIT) {
|
||||
log(` (dry-run) would commit ${r.file} — reverting workspace change`);
|
||||
await revertFile(r.file);
|
||||
await auditLog({
|
||||
action: "dry_run_would_commit",
|
||||
file: r.file,
|
||||
patches_applied: applied,
|
||||
patches_rejected: rejected.length,
|
||||
confidence_avg: r.confidence_avg,
|
||||
gradient_tier: r.gradient_tier,
|
||||
reviewer_model: r.accepted_model,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const ok = await gitCommit(r.file, applied_patches);
|
||||
if (ok) {
|
||||
committedFiles++;
|
||||
// Advance the warning baseline so a later patch on a different
|
||||
// file is compared against the new (committed) state, not the
|
||||
// pre-run state — otherwise every subsequent patch trips the gate.
|
||||
baselineWarnings = warnings;
|
||||
await auditLog({
|
||||
action: "committed",
|
||||
file: r.file,
|
||||
patches_applied: applied,
|
||||
patches_rejected: rejected.length,
|
||||
confidence_avg: r.confidence_avg,
|
||||
gradient_tier: r.gradient_tier,
|
||||
reviewer_model: r.accepted_model,
|
||||
warnings_after: warnings,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
log(`DONE · committed=${committedFiles} build_red=${revertedFiles} warning_revert=${warningReverts} rationale_revert=${rationaleReverts}`);
|
||||
}
|
||||
|
||||
await main();
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user