v1/mode: override knobs + staffing native runner + pass 2/3/4 harnesses
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts

Setup for the corpus-tightening experiment sweep (J 2026-04-26 — "now
is the only cheap window before the corpus gets large and refactoring
costs go up").

Override params on /v1/mode/execute (additive — old callers unaffected):
  force_matrix_corpus      — Pass 2: try alternate corpora per call
  force_relevance_threshold — Pass 2: sweep filter strictness
  force_temperature         — Pass 3: variance test

New native mode `staffing_inference_lakehouse` (Pass 4):
  - Same composer architecture as codereview_lakehouse
  - Staffing framing: coordinator producing fillable|contingent|
    unfillable verdict + ranked candidate list with playbook citations
  - matrix_corpus = workers_500k_v8
  - Validates that modes-as-prompt-molders generalizes beyond code
  - Framing explicitly says "do NOT fabricate workers" — the staffing
    analog of the lakehouse mode's symbol-grounding requirement

Three sweep harnesses:
  scripts/mode_pass2_corpus_sweep.ts — 4 corpora × 4 thresholds × 5 files
  scripts/mode_pass3_variance.ts     — 3 files × 3 temps × 5 reps
  scripts/mode_pass4_staffing.ts     — 5 fill requests through staffing mode

Each appends per-call rows to data/_kb/mode_experiments.jsonl which
mode_compare.ts already aggregates with grounding column.

Pass 1 (10 files × 5 modes broad sweep) currently running via the
existing scripts/mode_experiment.ts — gateway restart deferred until
it completes so the new override knobs aren't enabled mid-experiment.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-26 01:55:12 -05:00
co-authored by Claude Opus 4.7
parent 52bb216c2d
commit 56bf30cfd8
5 changed files with 408 additions and 5 deletions
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env bun
/**
* Pass 4: staffing_inference_lakehouse cross-domain validation.
*
* Runs the staffing-domain mode against synthetic fill requests.
* Validates that the modes-as-prompt-molders architecture generalizes
* beyond code review — the composer pattern (file_content + bug
* fingerprints + relevance-filtered matrix + domain framing) should
* produce grounded staffing recommendations the same way it produces
* grounded code reviews.
*
* Each fill request is posted as `file_content` (since the runner's
* shape expects file content; for staffing it's the request payload).
* file_path is set to a synthetic path under requests/ so pathway
* memory bucketing groups requests by geo+role.
*
* Usage: bun run scripts/mode_pass4_staffing.ts
*/
const GATEWAY = process.env.LH_GATEWAY ?? "http://localhost:3100";
const MODEL = process.env.LH_MODEL ?? "openai/gpt-oss-120b:free";
interface FillRequest {
city: string;
state: string;
role: string;
count: number;
deadline: string;
notes?: string;
}
const REQUESTS: FillRequest[] = [
{ city: "Toledo", state: "OH", role: "Welder", count: 2, deadline: "2026-04-29", notes: "OSHA 10 required" },
{ city: "Nashville", state: "TN", role: "Forklift Operator", count: 3, deadline: "2026-05-01" },
{ city: "Chicago", state: "IL", role: "Assembler", count: 5, deadline: "2026-04-30", notes: "second shift" },
{ city: "South Bend", state: "IN", role: "Electrician", count: 1, deadline: "2026-04-28", notes: "journeyman license" },
{ city: "Murfreesboro", state: "TN", role: "Packaging Operator", count: 4, deadline: "2026-05-02" },
];
function requestToPayload(req: FillRequest): string {
return [
`# Fill Request`,
`Role: ${req.role} × ${req.count}`,
`Location: ${req.city}, ${req.state}`,
`Deadline: ${req.deadline}`,
req.notes ? `Notes: ${req.notes}` : "",
"",
"Recommend candidates from the matrix data. Cite playbook references.",
].filter(Boolean).join("\n");
}
interface Result {
req: FillRequest;
ok: boolean;
response_chars?: number;
bug_fingerprints?: number;
matrix_kept?: number;
matrix_dropped?: number;
latency_ms?: number;
error?: string;
preview?: string;
}
async function runOne(req: FillRequest): Promise<Result> {
const payload = requestToPayload(req);
const file_path = `requests/${req.role.toLowerCase().replace(/\s+/g, "_")}_${req.city.toLowerCase().replace(/\s+/g, "_")}_${req.state}.md`;
try {
const r = await fetch(`${GATEWAY}/v1/mode/execute`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
task_class: "staffing_inference",
file_path,
file_content: payload,
force_mode: "staffing_inference_lakehouse",
force_model: MODEL,
}),
signal: AbortSignal.timeout(180_000),
});
if (!r.ok) {
const body = await r.text().catch(() => "");
return { req, ok: false, error: `HTTP ${r.status}: ${body.slice(0, 200)}` };
}
const j: any = await r.json();
return {
req, ok: true,
response_chars: (j.response ?? "").length,
bug_fingerprints: j.sources?.bug_fingerprints_count,
matrix_kept: j.sources?.matrix_chunks_kept,
matrix_dropped: j.sources?.matrix_chunks_dropped,
latency_ms: j.latency_ms,
preview: (j.response ?? "").slice(0, 400),
};
} catch (e: any) {
return { req, ok: false, error: e.message };
}
}
async function main() {
console.log(`[pass4] requests=${REQUESTS.length} model=${MODEL} mode=staffing_inference_lakehouse\n`);
let i = 0;
const results: Result[] = [];
for (const req of REQUESTS) {
i++;
process.stdout.write(` [${i}/${REQUESTS.length}] ${req.role.padEnd(22)} × ${req.count} in ${req.city}, ${req.state} ... `);
const r = await runOne(req);
results.push(r);
if (r.ok) {
console.log(`✓ resp=${r.response_chars} bug=${r.bug_fingerprints ?? 0} mtx=${r.matrix_kept ?? 0}/${(r.matrix_kept ?? 0) + (r.matrix_dropped ?? 0)} ${((r.latency_ms ?? 0) / 1000).toFixed(1)}s`);
} else {
console.log(`✗ ${r.error}`);
}
}
console.log(`\n[pass4] complete · ${results.filter(r => r.ok).length}/${results.length} succeeded\n`);
// Show first successful response head to verify the framing actually
// produced staffing-style output (verdict + ranked candidates) not
// generic prose.
const first = results.find(r => r.ok && r.preview);
if (first) {
console.log(`[pass4] first successful response preview (${first.req.city} ${first.req.role}):`);
console.log(first.preview!.split("\n").map(l => " | " + l).join("\n"));
}
}
main().catch(e => { console.error(e); process.exit(1); });