Auditor self-test branch: real-world pipelines + cohesion Phase C + KB index (PR #8)

Bundles 12 commits validating the auditor + scrum_master architecture end-to-end:

- enrich_prd_pipeline / hard_task_escalation / scrum_master_pipeline stress tests
- Tree-split + scrum_reviews.jsonl + kb_query surfacing
- Verdict → audit_lessons feedback loop (closed)
- kb_index aggregator with confidence-based severity policy
- 9-run + 5-run empirical tests proved the predictive-compounding property
- Level 1 correction: temp=0 cloud inference for deterministic per-claim verdicts
- audit_one.ts dry-run CLI
- Fixes: static quoted-string guard, empirical-claim classification, symbol-resolver gate, repo-file size cap

See PR #8 for run-by-run commit history.

auditor/checks/inference.ts

@@ -13,6 +13,8 @@
 // with a 15KB diff + claim list).
 
 import type { Claim, Finding } from "../types.ts";
+import { Glob } from "bun";
+import { readFile } from "node:fs/promises";
 
 const GATEWAY = process.env.LH_GATEWAY_URL ?? "http://localhost:3100";
 const MODEL = process.env.LH_AUDITOR_REVIEW_MODEL ?? "gpt-oss:120b";
@@ -22,6 +24,7 @@ const MODEL = process.env.LH_AUDITOR_REVIEW_MODEL ?? "gpt-oss:120b";
 // block finding when the file was simply outside the truncation window.
 const MAX_DIFF_CHARS = 40000;
 const CALL_TIMEOUT_MS = 120_000;
+const REPO_ROOT = "/home/profit/lakehouse";
 
 export async function runInferenceCheck(claims: Claim[], diff: string): Promise<Finding[]> {
   if (claims.length === 0) {
@@ -33,6 +36,21 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
     }];
   }
 
+  // Empirical claims (runtime metrics / observed outcomes) can't be
+  // verified from the diff. Drop them from the cloud prompt so the
+  // reviewer doesn't chase ghosts. A future `runtime_evidence` check
+  // can validate these against data/_kb/*/summary.json outputs.
+  const verifiable = claims.filter(c => c.strength !== "empirical");
+  const empiricalCount = claims.length - verifiable.length;
+  if (verifiable.length === 0) {
+    return [{
+      check: "inference",
+      severity: "info",
+      summary: `all ${claims.length} claims are empirical (runtime metrics) — skipping cloud inference`,
+      evidence: [`empirical claims can't be verified from a static diff; needs runtime-evidence check`],
+    }];
+  }
+
   const truncated = diff.length > MAX_DIFF_CHARS
     ? diff.slice(0, MAX_DIFF_CHARS) + `\n...[${diff.length - MAX_DIFF_CHARS} more chars truncated]`
     : diff;
@@ -70,7 +88,7 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
 
   const userMsg = [
     `Ship-claims the author made (numbered 0..N-1):`,
-    claims.map((c, i) => `  ${i}. [${c.strength}] "${c.text}" at ${c.location}`).join("\n"),
+    verifiable.map((c, i) => `  ${i}. [${c.strength}] "${c.text}" at ${c.location}`).join("\n"),
     "",
     `Diff:`,
     "```",
@@ -94,9 +112,22 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
           { role: "system", content: systemMsg },
           { role: "user", content: userMsg },
         ],
+        // Deterministic classification — temp=0 is greedy-sample, so
+        // identical input yields identical output on the same model
+        // version. This kills the signature creep we observed in the
+        // 9-run empirical test (sig_count 16→27 from cloud phrasing
+        // variance at temp=0.2).
+        //
+        // IMPORTANT: keep think=true. gpt-oss:120b is a reasoning
+        // model; setting think=false caused it to return empty content
+        // on large prompts (observed during Level 1 validation: 13421
+        // tokens used, empty content returned). The reasoning trace is
+        // variable prose, but at temp=0 the FINAL classification is
+        // still deterministic because greedy sampling converges to
+        // the same conclusion from the same starting state.
         max_tokens: 3000,
-        temperature: 0.2,
-        think: true, // T3 overseer should reason — JSON shape is still required
+        temperature: 0,
+        think: true,
       }),
       signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
     });
@@ -152,7 +183,9 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
   for (const v of parsed.claim_verdicts ?? []) {
     if (v?.backed === false) {
       const idx = typeof v.claim_idx === "number" ? v.claim_idx : -1;
-      const claim = claims[idx];
+      // Indices point at the verifiable[] list we sent the cloud,
+      // not the full claims[] list. Translate back.
+      const claim = verifiable[idx];
       if (!claim) continue;
       // Strong+unbacked = BLOCK. That's the whole point of the auditor.
       const sev: Finding["severity"] = claim.strength === "strong" ? "block"
@@ -172,17 +205,113 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
   }
 
   for (const g of parsed.unflagged_gaps ?? []) {
+    const summary = String(g?.summary ?? "?");
+    const location = String(g?.location ?? "?");
+    // False-positive guard — when the cloud says "X not defined in this
+    // diff" or "missing implementation of X", the cloud may just mean
+    // "X is not in the added lines," not "X doesn't exist in the repo."
+    // Extract candidate symbol names and grep the repo. If any symbol
+    // is defined elsewhere, drop the finding — it's a known-symbol
+    // reference, not a placeholder.
+    if (/not\s+defined|missing\s+implementation|never\s+referenced\s+or\s+integrated/i.test(summary)) {
+      const symbols = extractSymbols(summary);
+      if (symbols.length > 0) {
+        const resolved = await symbolsExistInRepo(symbols);
+        if (resolved.length === symbols.length) {
+          // Every named symbol exists somewhere in the repo — silent drop.
+          continue;
+        }
+        if (resolved.length > 0) {
+          // Partially resolved — demote to info with a note.
+          findings.push({
+            check: "inference",
+            severity: "info",
+            summary: `cloud gap partially resolved by repo grep: ${summary.slice(0, 120)}`,
+            evidence: [
+              `location: ${location.slice(0, 140)}`,
+              `resolved via grep: ${resolved.join(",")}`,
+              `unresolved: ${symbols.filter(s => !resolved.includes(s)).join(",")}`,
+            ],
+          });
+          continue;
+        }
+      }
+    }
     findings.push({
       check: "inference",
       severity: "warn",
-      summary: `cloud-flagged gap not in any claim: ${String(g?.summary ?? "?").slice(0, 120)}`,
-      evidence: [`location: ${String(g?.location ?? "?").slice(0, 140)}`],
+      summary: `cloud-flagged gap not in any claim: ${summary.slice(0, 120)}`,
+      evidence: [`location: ${location.slice(0, 140)}`],
     });
   }
 
   return findings;
 }
 
+// Pull out plausible code-symbol names from a summary string.
+// Matches:
+//   - identifier with backticks: `foo_bar`
+//   - identifier followed by parens: foo_bar()
+//   - CamelCase types
+//   - snake_case_functions
+// Filters out common English words that could be matched accidentally.
+const STOPWORDS = new Set([
+  "not","the","and","for","this","that","with","but","are","was","has",
+  "have","been","any","missing","implementation","diff","defined","never",
+  "referenced","integrated","flow","code","file","some","only","when",
+]);
+function extractSymbols(text: string): string[] {
+  const out = new Set<string>();
+  // `backticked` symbols
+  for (const m of text.matchAll(/`([A-Za-z_][A-Za-z0-9_]{2,})`/g)) out.add(m[1]);
+  // foo() or foo_bar() calls
+  for (const m of text.matchAll(/\b([A-Za-z_][A-Za-z0-9_]{2,})\s*\(/g)) out.add(m[1]);
+  // CamelCase types (3+ chars, must start with uppercase)
+  for (const m of text.matchAll(/\b([A-Z][A-Za-z0-9]{2,})\b/g)) out.add(m[1]);
+  return Array.from(out).filter(s => !STOPWORDS.has(s.toLowerCase()));
+}
+
+// Scan the repo for at least one definition of each symbol. Uses Bun's
+// Glob to walk TS/Rust/Python/JS sources; ignores node_modules, data/,
+// and target/. Skips files > 500KB — those are fixtures/snapshots that
+// won't contain a definition line and slurping them slows the audit.
+async function symbolsExistInRepo(symbols: string[]): Promise<string[]> {
+  const patterns = ["**/*.ts", "**/*.tsx", "**/*.rs", "**/*.py", "**/*.js"];
+  const skip = (p: string) => p.includes("/node_modules/") || p.startsWith("data/") || p.includes("/target/") || p.startsWith("dist/");
+  const MAX_FILE_BYTES = 500_000;
+  const { stat } = await import("node:fs/promises");
+  const resolved = new Set<string>();
+  const toFind = new Set(symbols);
+  for (const pat of patterns) {
+    if (toFind.size === 0) break;
+    const glob = new Glob(pat);
+    for await (const f of glob.scan({ cwd: REPO_ROOT, onlyFiles: true })) {
+      if (skip(f)) continue;
+      try { const s = await stat(`${REPO_ROOT}/${f}`); if (s.size > MAX_FILE_BYTES) continue; } catch { continue; }
+      let content: string;
+      try { content = await readFile(`${REPO_ROOT}/${f}`, "utf8"); } catch { continue; }
+      for (const sym of Array.from(toFind)) {
+        // Definition heuristics: `function sym`, `fn sym`, `const sym`,
+        // `let sym`, `def sym`, `class sym`, `struct sym`, `enum sym`,
+        // `trait sym`, `async function sym`, `pub (async )?fn sym`.
+        const re = new RegExp(
+          `\\b(function|async\\s+function|const|let|var|def|class|struct|enum|trait|impl|type|interface|fn|pub\\s+(async\\s+)?fn)\\s+${escapeRe(sym)}\\b`
+        );
+        if (re.test(content)) {
+          resolved.add(sym);
+          toFind.delete(sym);
+          if (toFind.size === 0) break;
+        }
+      }
+    }
+  }
+  return Array.from(resolved);
+}
+
+function escapeRe(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
 // Lift the first balanced JSON object out of the response. Tolerates
 // leading prose, code fences, and model reasoning preamble when the
 // cloud model ignored "strict JSON only."

auditor/checks/kb_query.ts

@@ -8,6 +8,7 @@
 // What this check reads (all file-backed, append-only or periodic):
 //   data/_kb/outcomes.jsonl         — per-scenario outcomes (kb.ts)
 //   data/_kb/error_corrections.jsonl — fail→succeed deltas on same sig
+//   data/_kb/scrum_reviews.jsonl     — scrum-master accepted reviews
 //   data/_observer/ops.jsonl         — observer ring → disk stream
 //   data/_bot/cycles/*.json          — bot cycle results
 //
@@ -17,14 +18,17 @@
 import { readFile, readdir, stat } from "node:fs/promises";
 import { join } from "node:path";
 import type { Claim, Finding } from "../types.ts";
+import { aggregate, ratingSeverity, formatAgg } from "../kb_index.ts";
 
 const KB_DIR = "/home/profit/lakehouse/data/_kb";
 const OBSERVER_OPS = "/home/profit/lakehouse/data/_observer/ops.jsonl";
 const BOT_CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
+const SCRUM_REVIEWS_JSONL = "/home/profit/lakehouse/data/_kb/scrum_reviews.jsonl";
+const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
 const TAIL_LINES = 500;
 const MAX_BOT_CYCLE_FILES = 30;
 
-export async function runKbCheck(claims: Claim[]): Promise<Finding[]> {
+export async function runKbCheck(claims: Claim[], prFiles: string[] = []): Promise<Finding[]> {
   const findings: Finding[] = [];
 
   // 1. Recent scenario outcomes: are strong-claim-style phrases showing
@@ -48,6 +52,27 @@ export async function runKbCheck(claims: Claim[]): Promise<Finding[]> {
   const obsFindings = await checkObserverStream();
   findings.push(...obsFindings);
 
+  // 5. Scrum-master reviews — surface prior accepted reviews for any
+  //    file in this PR's diff. Cohesion plan Phase C wire: the
+  //    auditor gets to "borrow" the scrum-master's deeper per-file
+  //    analysis instead of re-doing that work.
+  if (prFiles.length > 0) {
+    const scrumFindings = await checkScrumReviews(prFiles);
+    findings.push(...scrumFindings);
+  }
+
+  // 6. Audit-lessons feedback loop — summarize the top recurring
+  //    patterns from prior audits' block/warn findings. If the same
+  //    pattern signature has fired 3+ times across prior audits,
+  //    emit it as a block-severity finding so reviewers know this
+  //    is a known-recurring class, not a one-off. Does NOT couple
+  //    to the current audit's static/inference findings (those run
+  //    in parallel and we can't see them here) — the amplification
+  //    is emergent: if the current audit's finding-summary matches
+  //    a top recurrence, the reviewer sees both.
+  const auditLessonFindings = await checkAuditLessons();
+  findings.push(...auditLessonFindings);
+
   return findings;
 }
 
@@ -181,3 +206,90 @@ function observerBySource(ops: any[]): string {
   }
   return Object.entries(c).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(", ") || "empty";
 }
+
+// Audit-lessons — reads data/_kb/audit_lessons.jsonl (populated by
+// every audit's appendAuditLessons). Uses the shared kb_index
+// aggregator: groups by `signature`, distinct-scopes keyed by PR
+// number, severity from ratingSeverity(agg) which applies the
+// confidence × count rating (see kb_index.ts). This is the same
+// aggregation any other KB reader uses — shared discipline, not
+// per-check custom logic.
+async function checkAuditLessons(): Promise<Finding[]> {
+  const bySig = await aggregate<any>(AUDIT_LESSONS_JSONL, {
+    keyFn: (r) => r?.signature,
+    scopeFn: (r) => (r?.pr_number !== undefined ? `pr-${r.pr_number}` : undefined),
+    checkFn: (r) => r?.check,
+    tailLimit: TAIL_LINES * 4,
+  });
+  if (bySig.size === 0) return [];
+
+  const findings: Finding[] = [];
+  for (const [sig, agg] of bySig) {
+    // Silent on first-ever occurrence — not yet signal.
+    if (agg.count < 2) continue;
+    const sev = ratingSeverity(agg);
+    findings.push({
+      check: "kb_query",
+      severity: sev,
+      summary: `recurring audit pattern (${agg.distinct_scopes} distinct PRs, ${agg.count} flaggings, conf=${agg.confidence.toFixed(2)}): ${agg.representative_summary.slice(0, 160)}`,
+      evidence: [
+        `signature=${sig}`,
+        `checks: ${agg.checks.join(",")}`,
+        `scopes: ${agg.scopes.slice(-6).join(",")}`,
+        formatAgg(agg),
+      ],
+    });
+  }
+  return findings;
+}
+
+// Scrum-master reviews — the scrum pipeline writes one row per
+// accepted per-file review. We match reviews whose `file` matches
+// any path in the PR's diff, then surface the *preview* + which
+// model the escalation ladder had to reach. If the scrum-master
+// needed the 123B specialist or larger to resolve a file, that's
+// a meaningful signal about the code's complexity — and it's
+// surfaced to the PR without the auditor having to re-run the
+// escalation ladder itself.
+async function checkScrumReviews(prFiles: string[]): Promise<Finding[]> {
+  const rows = await tailJsonl<any>(SCRUM_REVIEWS_JSONL, TAIL_LINES);
+  if (rows.length === 0) return [];
+
+  // Match by exact file OR filename suffix — PR files arrive as
+  // `auditor/audit.ts`-style relative paths; scrum stores the same.
+  const norm = (p: string) => p.replace(/^\/+/, "").replace(/^home\/profit\/lakehouse\//, "");
+  const prSet = new Set(prFiles.map(norm));
+
+  // Keep only the most recent review per file (last-wins).
+  const latestByFile = new Map<string, any>();
+  for (const r of rows) {
+    const f = norm(String(r.file ?? ""));
+    if (!f) continue;
+    if (!prSet.has(f)) continue;
+    latestByFile.set(f, r);
+  }
+  if (latestByFile.size === 0) return [];
+
+  const findings: Finding[] = [];
+  for (const [file, r] of latestByFile) {
+    const model = String(r.accepted_model ?? "?");
+    const attempt = r.accepted_on_attempt ?? "?";
+    const treeSplit = !!r.tree_split_fired;
+    // Heuristic: if the scrum-master had to escalate past attempt 3,
+    // or had to tree-split, that's context the PR reviewer should see.
+    // Severity: info for low-escalation, warn if escalated far up
+    // the ladder (cloud specialist required).
+    const heavyEscalation = Number(attempt) >= 4;
+    const sev: "warn" | "info" = heavyEscalation ? "warn" : "info";
+    findings.push({
+      check: "kb_query",
+      severity: sev,
+      summary: `scrum-master review for \`${file}\` — accepted on attempt ${attempt} by \`${model}\`${treeSplit ? " (tree-split)" : ""}`,
+      evidence: [
+        `reviewed_at: ${r.reviewed_at ?? "?"}`,
+        `preview: ${String(r.suggestions_preview ?? "").slice(0, 300).replace(/\n/g, " ")}`,
+      ],
+    });
+  }
+  return findings;
+}

auditor/checks/static.ts

@@ -61,7 +61,13 @@ export function runStaticCheck(diff: string): Finding[] {
 
       if (!isAuditorCheckerFile) {
         for (const { re, why } of BLOCK_PATTERNS) {
-          if (re.test(added)) {
+          const m = added.match(re);
+          if (m && typeof m.index === "number") {
+            // Skip if the match sits inside a quoted string literal —
+            // this is how rubric files (tests/real-world/*, prompt
+            // templates) legitimately reference the patterns they
+            // guard against, without actually executing them.
+            if (isInsideQuotedString(added, m.index)) continue;
             findings.push({
               check: "static",
               severity: "block",
@@ -154,6 +160,25 @@ function extractNewFields(addedLines: string[]): string[] {
   return Array.from(fields);
 }
 
+// True if `pos` falls inside a double- or single-quoted string on this
+// line (backtick template literals too). Walks left→right toggling the
+// "in quote" state on each unescaped quote. Good enough for single-
+// line matches; multi-line strings aren't parsed (they're extremely
+// rare in the patterns we're blocking on, and would require a proper
+// tokenizer to handle correctly).
+function isInsideQuotedString(line: string, pos: number): boolean {
+  let inDouble = false, inSingle = false, inBacktick = false;
+  for (let i = 0; i < pos; i++) {
+    const c = line[i];
+    const esc = i > 0 && line[i - 1] === "\\";
+    if (esc) continue;
+    if (c === '"' && !inSingle && !inBacktick) inDouble = !inDouble;
+    else if (c === "'" && !inDouble && !inBacktick) inSingle = !inSingle;
+    else if (c === "`" && !inDouble && !inSingle) inBacktick = !inBacktick;
+  }
+  return inDouble || inSingle || inBacktick;
+}
+
 function escape(s: string): string {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }