v1/mode: model-aware enrichment downgrade + 3 corpora + variance harness
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
Pass 5 (5 reps × 4 conditions × 1 file on grok-4.1-fast) showed composing matrix corpora is anti-additive on strong models — composed lakehouse_arch + symbols LOST 5/5 head-to-head vs codereview_isolation (Δ −1.8 grounded findings, p=0.031). Default flips to isolation; matrix path now auto- downgrades when the resolved model is strong. Mode runner: - matrix_corpus is Vec<String> (string OR array via deserialize_string_or_vec) - top_k=6 from each corpus, merge by score, take top 8 globally - chunk tag prefers doc_id over source so reviewer sees [adr:009] vs [lakehouse_arch] - is_weak_model() gate auto-downgrades codereview_lakehouse → codereview_isolation for strong models (default-strong; weak = :free suffix or local last-resort) - LH_FORCE_FULL_ENRICHMENT=1 bypasses for diagnostic runs - EnrichmentSources.downgraded_from records when the gate fires Three corpora indexed via /vectors/index (5849 chunks total): - lakehouse_arch_v1 — ADRs + phases + PRD + scrum spec (93 docs, 2119 chunks) - scrum_findings_v1 — past scrum_reviews.jsonl (168 docs, 1260 chunks; EXCLUDED from defaults — 24% out-of-bounds line citations from cross-file drift) - lakehouse_symbols_v1 — regex-extracted pub items + /// docs (656 docs, 2470 chunks) Experiment infra: - scripts/build_*_corpus.ts — re-runnable when source content changes - scripts/mode_pass5_variance_paid.ts — N reps × M conditions on one file - scripts/mode_pass5_summarize.ts — mean ± σ + head-to-head, parser handles numbered + path-with-line + path-with-symbol finding tables - scripts/mode_compare.ts — groups by mode|corpus when sweeps span corpora - scripts/mode_experiment.ts — default model bumped to x-ai/grok-4.1-fast, --corpus flag for per-call override Decisions + open follow-ups: docs/MODE_RUNNER_TUNING_PLAN.md Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+21
-13
@@ -32,7 +32,7 @@ const DEFAULT_FILES = [
|
||||
"crates/queryd/src/service.rs",
|
||||
];
|
||||
|
||||
function parseArgs(): { files: string[]; modes: string[]; model: string } {
|
||||
function parseArgs(): { files: string[]; modes: string[]; model: string; corpus: string[] } {
|
||||
const args = Bun.argv.slice(2);
|
||||
const out: Record<string, string> = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
@@ -41,8 +41,12 @@ function parseArgs(): { files: string[]; modes: string[]; model: string } {
|
||||
}
|
||||
const files = (out.files ?? DEFAULT_FILES.join(",")).split(",").map(s => s.trim()).filter(Boolean);
|
||||
const modes = (out.modes ?? ALL_MODES.join(",")).split(",").map(s => s.trim()).filter(Boolean);
|
||||
const model = out.model ?? "openai/gpt-oss-120b:free";
|
||||
return { files, modes, model };
|
||||
// Default to the paid OpenRouter primary (matches scrum_master_pipeline
|
||||
// ladder rung 1). Pass `--model openai/gpt-oss-120b:free` if you want
|
||||
// the old free-tier baseline. See SCRUM_MASTER_SPEC.md for the ladder.
|
||||
const model = out.model ?? "x-ai/grok-4.1-fast";
|
||||
const corpus = (out.corpus ?? "").split(",").map(s => s.trim()).filter(Boolean);
|
||||
return { files, modes, model, corpus };
|
||||
}
|
||||
|
||||
interface RunResult {
|
||||
@@ -58,19 +62,22 @@ interface RunResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
async function runOne(file: string, mode: string, model: string): Promise<RunResult> {
|
||||
async function runOne(file: string, mode: string, model: string, corpus: string[]): Promise<RunResult> {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const body: any = {
|
||||
task_class: TASK_CLASS,
|
||||
file_path: file,
|
||||
force_mode: mode,
|
||||
force_model: model,
|
||||
};
|
||||
if (corpus.length === 1) body.force_matrix_corpus = corpus[0];
|
||||
else if (corpus.length > 1) body.force_matrix_corpus = corpus;
|
||||
const r = await fetch(`${GATEWAY}/v1/mode/execute`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
task_class: TASK_CLASS,
|
||||
file_path: file,
|
||||
force_mode: mode,
|
||||
force_model: model,
|
||||
}),
|
||||
signal: AbortSignal.timeout(180_000),
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(240_000),
|
||||
});
|
||||
if (!r.ok) {
|
||||
const body = await r.text().catch(() => "");
|
||||
@@ -92,9 +99,10 @@ async function runOne(file: string, mode: string, model: string): Promise<RunRes
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { files, modes, model } = parseArgs();
|
||||
const { files, modes, model, corpus } = parseArgs();
|
||||
console.log(`[experiment] files=${files.length} × modes=${modes.length} = ${files.length * modes.length} runs`);
|
||||
console.log(`[experiment] model=${model} task=${TASK_CLASS} gateway=${GATEWAY}`);
|
||||
if (corpus.length > 0) console.log(`[experiment] corpus override: ${corpus.join(" + ")}`);
|
||||
console.log("");
|
||||
|
||||
const results: RunResult[] = [];
|
||||
@@ -103,7 +111,7 @@ async function main() {
|
||||
for (const mode of modes) {
|
||||
i++;
|
||||
process.stdout.write(` [${i}/${files.length * modes.length}] ${mode.padEnd(28)} ${file} ... `);
|
||||
const r = await runOne(file, mode, model);
|
||||
const r = await runOne(file, mode, model, corpus);
|
||||
results.push(r);
|
||||
if (r.ok) {
|
||||
console.log(
|
||||
|
||||
Reference in New Issue
Block a user