v1/mode: model-aware enrichment downgrade + 3 corpora + variance harness
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts

Pass 5 (5 reps × 4 conditions × 1 file on grok-4.1-fast) showed composing
matrix corpora is anti-additive on strong models — composed lakehouse_arch
+ symbols LOST 5/5 head-to-head vs codereview_isolation (Δ −1.8 grounded
findings, p=0.031). Default flips to isolation; matrix path now auto-
downgrades when the resolved model is strong.

Mode runner:
- matrix_corpus is Vec<String> (string OR array via deserialize_string_or_vec)
- top_k=6 from each corpus, merge by score, take top 8 globally
- chunk tag prefers doc_id over source so reviewer sees [adr:009] vs [lakehouse_arch]
- is_weak_model() gate auto-downgrades codereview_lakehouse → codereview_isolation
  for strong models (default-strong; weak = :free suffix or local last-resort)
- LH_FORCE_FULL_ENRICHMENT=1 bypasses for diagnostic runs
- EnrichmentSources.downgraded_from records when the gate fires

Three corpora indexed via /vectors/index (5849 chunks total):
- lakehouse_arch_v1 — ADRs + phases + PRD + scrum spec (93 docs, 2119 chunks)
- scrum_findings_v1 — past scrum_reviews.jsonl (168 docs, 1260 chunks; EXCLUDED
  from defaults — 24% out-of-bounds line citations from cross-file drift)
- lakehouse_symbols_v1 — regex-extracted pub items + /// docs (656 docs, 2470 chunks)

Experiment infra:
- scripts/build_*_corpus.ts — re-runnable when source content changes
- scripts/mode_pass5_variance_paid.ts — N reps × M conditions on one file
- scripts/mode_pass5_summarize.ts — mean ± σ + head-to-head, parser handles
  numbered + path-with-line + path-with-symbol finding tables
- scripts/mode_compare.ts — groups by mode|corpus when sweeps span corpora
- scripts/mode_experiment.ts — default model bumped to x-ai/grok-4.1-fast,
  --corpus flag for per-call override

Decisions + open follow-ups: docs/MODE_RUNNER_TUNING_PLAN.md

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-26 17:29:17 -05:00
co-authored by Claude Opus 4.7
parent 56bf30cfd8
commit 2dbc8dbc83
10 changed files with 1043 additions and 82 deletions
+21 -13
View File
@@ -32,7 +32,7 @@ const DEFAULT_FILES = [
"crates/queryd/src/service.rs",
];
function parseArgs(): { files: string[]; modes: string[]; model: string } {
function parseArgs(): { files: string[]; modes: string[]; model: string; corpus: string[] } {
const args = Bun.argv.slice(2);
const out: Record<string, string> = {};
for (let i = 0; i < args.length; i++) {
@@ -41,8 +41,12 @@ function parseArgs(): { files: string[]; modes: string[]; model: string } {
}
const files = (out.files ?? DEFAULT_FILES.join(",")).split(",").map(s => s.trim()).filter(Boolean);
const modes = (out.modes ?? ALL_MODES.join(",")).split(",").map(s => s.trim()).filter(Boolean);
const model = out.model ?? "openai/gpt-oss-120b:free";
return { files, modes, model };
// Default to the paid OpenRouter primary (matches scrum_master_pipeline
// ladder rung 1). Pass `--model openai/gpt-oss-120b:free` if you want
// the old free-tier baseline. See SCRUM_MASTER_SPEC.md for the ladder.
const model = out.model ?? "x-ai/grok-4.1-fast";
const corpus = (out.corpus ?? "").split(",").map(s => s.trim()).filter(Boolean);
return { files, modes, model, corpus };
}
interface RunResult {
@@ -58,19 +62,22 @@ interface RunResult {
error?: string;
}
async function runOne(file: string, mode: string, model: string): Promise<RunResult> {
async function runOne(file: string, mode: string, model: string, corpus: string[]): Promise<RunResult> {
const t0 = Date.now();
try {
const body: any = {
task_class: TASK_CLASS,
file_path: file,
force_mode: mode,
force_model: model,
};
if (corpus.length === 1) body.force_matrix_corpus = corpus[0];
else if (corpus.length > 1) body.force_matrix_corpus = corpus;
const r = await fetch(`${GATEWAY}/v1/mode/execute`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
task_class: TASK_CLASS,
file_path: file,
force_mode: mode,
force_model: model,
}),
signal: AbortSignal.timeout(180_000),
body: JSON.stringify(body),
signal: AbortSignal.timeout(240_000),
});
if (!r.ok) {
const body = await r.text().catch(() => "");
@@ -92,9 +99,10 @@ async function runOne(file: string, mode: string, model: string): Promise<RunRes
}
async function main() {
const { files, modes, model } = parseArgs();
const { files, modes, model, corpus } = parseArgs();
console.log(`[experiment] files=${files.length} × modes=${modes.length} = ${files.length * modes.length} runs`);
console.log(`[experiment] model=${model} task=${TASK_CLASS} gateway=${GATEWAY}`);
if (corpus.length > 0) console.log(`[experiment] corpus override: ${corpus.join(" + ")}`);
console.log("");
const results: RunResult[] = [];
@@ -103,7 +111,7 @@ async function main() {
for (const mode of modes) {
i++;
process.stdout.write(` [${i}/${files.length * modes.length}] ${mode.padEnd(28)} ${file} ... `);
const r = await runOne(file, mode, model);
const r = await runOne(file, mode, model, corpus);
results.push(r);
if (r.ok) {
console.log(