Auditor self-test branch: real-world pipelines + cohesion Phase C + KB index (PR #8)

Bundles 12 commits validating the auditor + scrum_master architecture end-to-end:

- enrich_prd_pipeline / hard_task_escalation / scrum_master_pipeline stress tests
- Tree-split + scrum_reviews.jsonl + kb_query surfacing
- Verdict → audit_lessons feedback loop (closed)
- kb_index aggregator with confidence-based severity policy
- 9-run + 5-run empirical tests proved the predictive-compounding property
- Level 1 correction: temp=0 cloud inference for deterministic per-claim verdicts
- audit_one.ts dry-run CLI
- Fixes: static quoted-string guard, empirical-claim classification, symbol-resolver gate, repo-file size cap

See PR #8 for run-by-run commit history.

auditor/README.md

@@ -33,6 +33,13 @@ Defaults: polls every 90s, stops on `auditor.paused` file present.
 
 - `data/_auditor/state.json` — last-audited head SHA per PR
 - `data/_auditor/verdicts/{pr}-{sha}.json` — per-run verdict record
+- `data/_kb/audit_lessons.jsonl` — one row per block/warn finding,
+  path-agnostic signature for dedup. Tailed by kb_query on each audit
+  to surface recurring patterns (2+ distinct PRs with same signature
+  → info, 3-4 → warn, 5+ → block). This is how the auditor learns.
+- `data/_kb/scrum_reviews.jsonl` — scrum-master per-file reviews. If
+  a file in the current PR has been scrum-reviewed, kb_query surfaces
+  the review as a finding with the accepted model and attempt count.
 
 ## Where YOU edit
 

auditor/audit.ts

@@ -12,7 +12,8 @@
 // review — reviews have self-review restrictions on Gitea and the
 // auditor currently uses the same PAT as the PR author).
 
-import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
 import { join } from "node:path";
 import type { PrSnapshot, Verdict, Finding } from "./types.ts";
 import { getPrDiff, postCommitStatus, postIssueComment } from "./gitea.ts";
@@ -24,6 +25,10 @@ import { runInferenceCheck } from "./checks/inference.ts";
 import { runKbCheck } from "./checks/kb_query.ts";
 
 const VERDICTS_DIR = "/home/profit/lakehouse/data/_auditor/verdicts";
+// Playbook for audit findings — one row per block/warn finding from a
+// verdict. kb_query tails this next audit and escalates recurrences.
+// Structured as JSONL so it's cheap to append and cheap to tail.
+const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
 
 export interface AuditOptions {
   // Skip the cloud inference call (fast path for iteration). Default false.
@@ -52,7 +57,7 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
     runStaticCheck(diff),
     opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
     opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff),
-    runKbCheck(claims),
+    runKbCheck(claims, pr.files.map(f => f.path)),
   ]);
 
   const allFindings: Finding[] = [
@@ -72,6 +77,7 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
     claims_strong: claims.filter(c => c.strength === "strong").length,
     claims_moderate: claims.filter(c => c.strength === "moderate").length,
     claims_weak: claims.filter(c => c.strength === "weak").length,
+    claims_empirical: claims.filter(c => c.strength === "empirical").length,
     claims_total: claims.length,
     diff_bytes: diff.length,
   };
@@ -80,6 +86,15 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
 
   await persistVerdict(verdict);
 
+  // Feedback loop — every block/warn finding becomes a row in
+  // audit_lessons.jsonl, dedup-keyed by (check, normalized-summary).
+  // The next audit's kb_query reads these and escalates recurring
+  // findings so we don't lose the "this pattern has been flagged
+  // before" signal across runs. Fire-and-forget; failure here must
+  // not break the audit.
+  appendAuditLessons(verdict).catch(e =>
+    console.error(`[audit] audit_lessons append failed: ${(e as Error).message}`));
+
   if (!opts.dry_run) {
     await postToGitea(verdict);
   }
@@ -87,6 +102,42 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
   return verdict;
 }
 
+// Normalizes a finding summary for dedup: strips path-specific tails
+// ("in path/to/file.ts" → "in <file>"), line numbers, and long
+// commit-hash snippets. The goal is: the SAME class of finding on
+// DIFFERENT files should share a signature, so we can measure
+// "this pattern keeps showing up."
+function normalizedSignature(f: Finding): string {
+  const summary = String(f.summary)
+    .replace(/\bin\s+\S+\.(ts|rs|js|py|md)\b/gi, "in <file>")
+    .replace(/:\+?\d+\b/g, ":<line>")
+    .replace(/[0-9a-f]{8,}/gi, "<hash>")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 240);
+  const src = `${f.check}::${f.severity}::${summary}`;
+  return createHash("sha256").update(src).digest("hex").slice(0, 16);
+}
+
+async function appendAuditLessons(v: Verdict): Promise<void> {
+  const actionable = v.findings.filter(f => f.severity === "block" || f.severity === "warn");
+  if (actionable.length === 0) return;
+  await mkdir(join(AUDIT_LESSONS_JSONL, ".."), { recursive: true });
+  const rows: string[] = [];
+  for (const f of actionable) {
+    rows.push(JSON.stringify({
+      signature: normalizedSignature(f),
+      check: f.check,
+      severity: f.severity,
+      summary: f.summary,
+      pr_number: v.pr_number,
+      head_sha: v.head_sha,
+      audited_at: v.audited_at,
+    }));
+  }
+  await appendFile(AUDIT_LESSONS_JSONL, rows.join("\n") + "\n");
+}
+
 async function persistVerdict(v: Verdict): Promise<void> {
   await mkdir(VERDICTS_DIR, { recursive: true });
   const filename = `${v.pr_number}-${v.head_sha.slice(0, 12)}.json`;

auditor/audit_one.ts

@@ -0,0 +1,68 @@
+// One-shot dry-run audit of a single PR. Useful for verifying check
+// behavior (kb_query scrum surfacing, inference prompts, etc.) without
+// posting to Gitea. Does NOT touch state.json and does NOT post
+// commit status or PR comments.
+//
+// Run: bun run auditor/audit_one.ts <pr-number>
+
+import { getPrSnapshot } from "./gitea.ts";
+import { auditPr } from "./audit.ts";
+
+async function main() {
+  const prNumRaw = process.argv[2];
+  if (!prNumRaw) {
+    console.error("usage: bun run auditor/audit_one.ts <pr-number>");
+    process.exit(2);
+  }
+  const prNum = Number(prNumRaw);
+  if (!Number.isFinite(prNum)) {
+    console.error(`invalid PR number: ${prNumRaw}`);
+    process.exit(2);
+  }
+
+  console.log(`[audit_one] fetching PR #${prNum}...`);
+  const pr = await getPrSnapshot(prNum);
+  console.log(`[audit_one] PR #${pr.number}: "${pr.title}" (head=${pr.head_sha.slice(0, 12)})`);
+  console.log(`[audit_one] files in diff: ${pr.files.length}`);
+  for (const f of pr.files) console.log(`  - ${f.path} (+${f.additions}/-${f.deletions})`);
+  console.log("");
+
+  const verdict = await auditPr(pr, {
+    dry_run: true,                      // no Gitea posting
+    skip_dynamic: true,                  // don't run fixture
+    skip_inference: process.env.LH_AUDITOR_SKIP_INFERENCE === "1",
+  });
+
+  console.log("\n═══ VERDICT ═══");
+  console.log(`overall:  ${verdict.overall}`);
+  console.log(`one-liner: ${verdict.one_liner}`);
+  console.log(`findings:  total=${verdict.metrics.findings_total} block=${verdict.metrics.findings_block} warn=${verdict.metrics.findings_warn} info=${verdict.metrics.findings_info}`);
+  console.log("");
+
+  // Print findings, highlighting kb_query scrum surfacing
+  const byCheck: Record<string, typeof verdict.findings> = {};
+  for (const f of verdict.findings) (byCheck[f.check] ||= []).push(f);
+
+  for (const [check, findings] of Object.entries(byCheck)) {
+    console.log(`── ${check} (${findings.length}) ──`);
+    for (const f of findings) {
+      const tag = f.severity === "block" ? "🛑" : f.severity === "warn" ? "⚠️ " : "ℹ️ ";
+      console.log(`  ${tag} [${f.severity}] ${f.summary}`);
+      if (f.summary.includes("scrum-master")) {
+        for (const e of f.evidence) {
+          console.log(`      → ${e.slice(0, 200)}`);
+        }
+      }
+    }
+  }
+
+  const scrumFindings = verdict.findings.filter(f => f.summary.includes("scrum-master"));
+  console.log("");
+  console.log(`═══ SCRUM WIRE CHECK: ${scrumFindings.length} scrum-master findings surfaced by kb_query ═══`);
+  if (scrumFindings.length === 0) {
+    console.log("  (none — either no matching scrum_reviews.jsonl rows, or files didn't match PR diff)");
+  }
+  process.exit(0);
+}
+
+main().catch(e => { console.error("[audit_one] fatal:", e); process.exit(1); });

auditor/checks/inference.ts

@@ -13,6 +13,8 @@
 // with a 15KB diff + claim list).
 
 import type { Claim, Finding } from "../types.ts";
+import { Glob } from "bun";
+import { readFile } from "node:fs/promises";
 
 const GATEWAY = process.env.LH_GATEWAY_URL ?? "http://localhost:3100";
 const MODEL = process.env.LH_AUDITOR_REVIEW_MODEL ?? "gpt-oss:120b";
@@ -22,6 +24,7 @@ const MODEL = process.env.LH_AUDITOR_REVIEW_MODEL ?? "gpt-oss:120b";
 // block finding when the file was simply outside the truncation window.
 const MAX_DIFF_CHARS = 40000;
 const CALL_TIMEOUT_MS = 120_000;
+const REPO_ROOT = "/home/profit/lakehouse";
 
 export async function runInferenceCheck(claims: Claim[], diff: string): Promise<Finding[]> {
   if (claims.length === 0) {
@@ -33,6 +36,21 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
     }];
   }
 
+  // Empirical claims (runtime metrics / observed outcomes) can't be
+  // verified from the diff. Drop them from the cloud prompt so the
+  // reviewer doesn't chase ghosts. A future `runtime_evidence` check
+  // can validate these against data/_kb/*/summary.json outputs.
+  const verifiable = claims.filter(c => c.strength !== "empirical");
+  const empiricalCount = claims.length - verifiable.length;
+  if (verifiable.length === 0) {
+    return [{
+      check: "inference",
+      severity: "info",
+      summary: `all ${claims.length} claims are empirical (runtime metrics) — skipping cloud inference`,
+      evidence: [`empirical claims can't be verified from a static diff; needs runtime-evidence check`],
+    }];
+  }
+
   const truncated = diff.length > MAX_DIFF_CHARS
     ? diff.slice(0, MAX_DIFF_CHARS) + `\n...[${diff.length - MAX_DIFF_CHARS} more chars truncated]`
     : diff;
@@ -70,7 +88,7 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
 
   const userMsg = [
     `Ship-claims the author made (numbered 0..N-1):`,
-    claims.map((c, i) => `  ${i}. [${c.strength}] "${c.text}" at ${c.location}`).join("\n"),
+    verifiable.map((c, i) => `  ${i}. [${c.strength}] "${c.text}" at ${c.location}`).join("\n"),
     "",
     `Diff:`,
     "```",
@@ -94,9 +112,22 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
           { role: "system", content: systemMsg },
           { role: "user", content: userMsg },
         ],
+        // Deterministic classification — temp=0 is greedy-sample, so
+        // identical input yields identical output on the same model
+        // version. This kills the signature creep we observed in the
+        // 9-run empirical test (sig_count 16→27 from cloud phrasing
+        // variance at temp=0.2).
+        //
+        // IMPORTANT: keep think=true. gpt-oss:120b is a reasoning
+        // model; setting think=false caused it to return empty content
+        // on large prompts (observed during Level 1 validation: 13421
+        // tokens used, empty content returned). The reasoning trace is
+        // variable prose, but at temp=0 the FINAL classification is
+        // still deterministic because greedy sampling converges to
+        // the same conclusion from the same starting state.
         max_tokens: 3000,
-        temperature: 0.2,
-        think: true, // T3 overseer should reason — JSON shape is still required
+        temperature: 0,
+        think: true,
       }),
       signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
     });
@@ -152,7 +183,9 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
   for (const v of parsed.claim_verdicts ?? []) {
     if (v?.backed === false) {
       const idx = typeof v.claim_idx === "number" ? v.claim_idx : -1;
-      const claim = claims[idx];
+      // Indices point at the verifiable[] list we sent the cloud,
+      // not the full claims[] list. Translate back.
+      const claim = verifiable[idx];
       if (!claim) continue;
       // Strong+unbacked = BLOCK. That's the whole point of the auditor.
       const sev: Finding["severity"] = claim.strength === "strong" ? "block"
@@ -172,17 +205,113 @@ export async function runInferenceCheck(claims: Claim[], diff: string): Promise<
   }
 
   for (const g of parsed.unflagged_gaps ?? []) {
+    const summary = String(g?.summary ?? "?");
+    const location = String(g?.location ?? "?");
+    // False-positive guard — when the cloud says "X not defined in this
+    // diff" or "missing implementation of X", the cloud may just mean
+    // "X is not in the added lines," not "X doesn't exist in the repo."
+    // Extract candidate symbol names and grep the repo. If any symbol
+    // is defined elsewhere, drop the finding — it's a known-symbol
+    // reference, not a placeholder.
+    if (/not\s+defined|missing\s+implementation|never\s+referenced\s+or\s+integrated/i.test(summary)) {
+      const symbols = extractSymbols(summary);
+      if (symbols.length > 0) {
+        const resolved = await symbolsExistInRepo(symbols);
+        if (resolved.length === symbols.length) {
+          // Every named symbol exists somewhere in the repo — silent drop.
+          continue;
+        }
+        if (resolved.length > 0) {
+          // Partially resolved — demote to info with a note.
+          findings.push({
+            check: "inference",
+            severity: "info",
+            summary: `cloud gap partially resolved by repo grep: ${summary.slice(0, 120)}`,
+            evidence: [
+              `location: ${location.slice(0, 140)}`,
+              `resolved via grep: ${resolved.join(",")}`,
+              `unresolved: ${symbols.filter(s => !resolved.includes(s)).join(",")}`,
+            ],
+          });
+          continue;
+        }
+      }
+    }
     findings.push({
       check: "inference",
       severity: "warn",
-      summary: `cloud-flagged gap not in any claim: ${String(g?.summary ?? "?").slice(0, 120)}`,
-      evidence: [`location: ${String(g?.location ?? "?").slice(0, 140)}`],
+      summary: `cloud-flagged gap not in any claim: ${summary.slice(0, 120)}`,
+      evidence: [`location: ${location.slice(0, 140)}`],
     });
   }
 
   return findings;
 }
 
+// Pull out plausible code-symbol names from a summary string.
+// Matches:
+//   - identifier with backticks: `foo_bar`
+//   - identifier followed by parens: foo_bar()
+//   - CamelCase types
+//   - snake_case_functions
+// Filters out common English words that could be matched accidentally.
+const STOPWORDS = new Set([
+  "not","the","and","for","this","that","with","but","are","was","has",
+  "have","been","any","missing","implementation","diff","defined","never",
+  "referenced","integrated","flow","code","file","some","only","when",
+]);
+function extractSymbols(text: string): string[] {
+  const out = new Set<string>();
+  // `backticked` symbols
+  for (const m of text.matchAll(/`([A-Za-z_][A-Za-z0-9_]{2,})`/g)) out.add(m[1]);
+  // foo() or foo_bar() calls
+  for (const m of text.matchAll(/\b([A-Za-z_][A-Za-z0-9_]{2,})\s*\(/g)) out.add(m[1]);
+  // CamelCase types (3+ chars, must start with uppercase)
+  for (const m of text.matchAll(/\b([A-Z][A-Za-z0-9]{2,})\b/g)) out.add(m[1]);
+  return Array.from(out).filter(s => !STOPWORDS.has(s.toLowerCase()));
+}
+
+// Scan the repo for at least one definition of each symbol. Uses Bun's
+// Glob to walk TS/Rust/Python/JS sources; ignores node_modules, data/,
+// and target/. Skips files > 500KB — those are fixtures/snapshots that
+// won't contain a definition line and slurping them slows the audit.
+async function symbolsExistInRepo(symbols: string[]): Promise<string[]> {
+  const patterns = ["**/*.ts", "**/*.tsx", "**/*.rs", "**/*.py", "**/*.js"];
+  const skip = (p: string) => p.includes("/node_modules/") || p.startsWith("data/") || p.includes("/target/") || p.startsWith("dist/");
+  const MAX_FILE_BYTES = 500_000;
+  const { stat } = await import("node:fs/promises");
+  const resolved = new Set<string>();
+  const toFind = new Set(symbols);
+  for (const pat of patterns) {
+    if (toFind.size === 0) break;
+    const glob = new Glob(pat);
+    for await (const f of glob.scan({ cwd: REPO_ROOT, onlyFiles: true })) {
+      if (skip(f)) continue;
+      try { const s = await stat(`${REPO_ROOT}/${f}`); if (s.size > MAX_FILE_BYTES) continue; } catch { continue; }
+      let content: string;
+      try { content = await readFile(`${REPO_ROOT}/${f}`, "utf8"); } catch { continue; }
+      for (const sym of Array.from(toFind)) {
+        // Definition heuristics: `function sym`, `fn sym`, `const sym`,
+        // `let sym`, `def sym`, `class sym`, `struct sym`, `enum sym`,
+        // `trait sym`, `async function sym`, `pub (async )?fn sym`.
+        const re = new RegExp(
+          `\\b(function|async\\s+function|const|let|var|def|class|struct|enum|trait|impl|type|interface|fn|pub\\s+(async\\s+)?fn)\\s+${escapeRe(sym)}\\b`
+        );
+        if (re.test(content)) {
+          resolved.add(sym);
+          toFind.delete(sym);
+          if (toFind.size === 0) break;
+        }
+      }
+    }
+  }
+  return Array.from(resolved);
+}
+
+function escapeRe(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
 // Lift the first balanced JSON object out of the response. Tolerates
 // leading prose, code fences, and model reasoning preamble when the
 // cloud model ignored "strict JSON only."

auditor/checks/kb_query.ts

@@ -8,6 +8,7 @@
 // What this check reads (all file-backed, append-only or periodic):
 //   data/_kb/outcomes.jsonl         — per-scenario outcomes (kb.ts)
 //   data/_kb/error_corrections.jsonl — fail→succeed deltas on same sig
+//   data/_kb/scrum_reviews.jsonl     — scrum-master accepted reviews
 //   data/_observer/ops.jsonl         — observer ring → disk stream
 //   data/_bot/cycles/*.json          — bot cycle results
 //
@@ -17,14 +18,17 @@
 import { readFile, readdir, stat } from "node:fs/promises";
 import { join } from "node:path";
 import type { Claim, Finding } from "../types.ts";
+import { aggregate, ratingSeverity, formatAgg } from "../kb_index.ts";
 
 const KB_DIR = "/home/profit/lakehouse/data/_kb";
 const OBSERVER_OPS = "/home/profit/lakehouse/data/_observer/ops.jsonl";
 const BOT_CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
+const SCRUM_REVIEWS_JSONL = "/home/profit/lakehouse/data/_kb/scrum_reviews.jsonl";
+const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
 const TAIL_LINES = 500;
 const MAX_BOT_CYCLE_FILES = 30;
 
-export async function runKbCheck(claims: Claim[]): Promise<Finding[]> {
+export async function runKbCheck(claims: Claim[], prFiles: string[] = []): Promise<Finding[]> {
   const findings: Finding[] = [];
 
   // 1. Recent scenario outcomes: are strong-claim-style phrases showing
@@ -48,6 +52,27 @@ export async function runKbCheck(claims: Claim[]): Promise<Finding[]> {
   const obsFindings = await checkObserverStream();
   findings.push(...obsFindings);
 
+  // 5. Scrum-master reviews — surface prior accepted reviews for any
+  //    file in this PR's diff. Cohesion plan Phase C wire: the
+  //    auditor gets to "borrow" the scrum-master's deeper per-file
+  //    analysis instead of re-doing that work.
+  if (prFiles.length > 0) {
+    const scrumFindings = await checkScrumReviews(prFiles);
+    findings.push(...scrumFindings);
+  }
+
+  // 6. Audit-lessons feedback loop — summarize the top recurring
+  //    patterns from prior audits' block/warn findings. If the same
+  //    pattern signature has fired 3+ times across prior audits,
+  //    emit it as a block-severity finding so reviewers know this
+  //    is a known-recurring class, not a one-off. Does NOT couple
+  //    to the current audit's static/inference findings (those run
+  //    in parallel and we can't see them here) — the amplification
+  //    is emergent: if the current audit's finding-summary matches
+  //    a top recurrence, the reviewer sees both.
+  const auditLessonFindings = await checkAuditLessons();
+  findings.push(...auditLessonFindings);
+
   return findings;
 }
 
@@ -181,3 +206,90 @@ function observerBySource(ops: any[]): string {
   }
   return Object.entries(c).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(", ") || "empty";
 }
+
+// Audit-lessons — reads data/_kb/audit_lessons.jsonl (populated by
+// every audit's appendAuditLessons). Uses the shared kb_index
+// aggregator: groups by `signature`, distinct-scopes keyed by PR
+// number, severity from ratingSeverity(agg) which applies the
+// confidence × count rating (see kb_index.ts). This is the same
+// aggregation any other KB reader uses — shared discipline, not
+// per-check custom logic.
+async function checkAuditLessons(): Promise<Finding[]> {
+  const bySig = await aggregate<any>(AUDIT_LESSONS_JSONL, {
+    keyFn: (r) => r?.signature,
+    scopeFn: (r) => (r?.pr_number !== undefined ? `pr-${r.pr_number}` : undefined),
+    checkFn: (r) => r?.check,
+    tailLimit: TAIL_LINES * 4,
+  });
+  if (bySig.size === 0) return [];
+
+  const findings: Finding[] = [];
+  for (const [sig, agg] of bySig) {
+    // Silent on first-ever occurrence — not yet signal.
+    if (agg.count < 2) continue;
+    const sev = ratingSeverity(agg);
+    findings.push({
+      check: "kb_query",
+      severity: sev,
+      summary: `recurring audit pattern (${agg.distinct_scopes} distinct PRs, ${agg.count} flaggings, conf=${agg.confidence.toFixed(2)}): ${agg.representative_summary.slice(0, 160)}`,
+      evidence: [
+        `signature=${sig}`,
+        `checks: ${agg.checks.join(",")}`,
+        `scopes: ${agg.scopes.slice(-6).join(",")}`,
+        formatAgg(agg),
+      ],
+    });
+  }
+  return findings;
+}
+
+// Scrum-master reviews — the scrum pipeline writes one row per
+// accepted per-file review. We match reviews whose `file` matches
+// any path in the PR's diff, then surface the *preview* + which
+// model the escalation ladder had to reach. If the scrum-master
+// needed the 123B specialist or larger to resolve a file, that's
+// a meaningful signal about the code's complexity — and it's
+// surfaced to the PR without the auditor having to re-run the
+// escalation ladder itself.
+async function checkScrumReviews(prFiles: string[]): Promise<Finding[]> {
+  const rows = await tailJsonl<any>(SCRUM_REVIEWS_JSONL, TAIL_LINES);
+  if (rows.length === 0) return [];
+
+  // Match by exact file OR filename suffix — PR files arrive as
+  // `auditor/audit.ts`-style relative paths; scrum stores the same.
+  const norm = (p: string) => p.replace(/^\/+/, "").replace(/^home\/profit\/lakehouse\//, "");
+  const prSet = new Set(prFiles.map(norm));
+
+  // Keep only the most recent review per file (last-wins).
+  const latestByFile = new Map<string, any>();
+  for (const r of rows) {
+    const f = norm(String(r.file ?? ""));
+    if (!f) continue;
+    if (!prSet.has(f)) continue;
+    latestByFile.set(f, r);
+  }
+  if (latestByFile.size === 0) return [];
+
+  const findings: Finding[] = [];
+  for (const [file, r] of latestByFile) {
+    const model = String(r.accepted_model ?? "?");
+    const attempt = r.accepted_on_attempt ?? "?";
+    const treeSplit = !!r.tree_split_fired;
+    // Heuristic: if the scrum-master had to escalate past attempt 3,
+    // or had to tree-split, that's context the PR reviewer should see.
+    // Severity: info for low-escalation, warn if escalated far up
+    // the ladder (cloud specialist required).
+    const heavyEscalation = Number(attempt) >= 4;
+    const sev: "warn" | "info" = heavyEscalation ? "warn" : "info";
+    findings.push({
+      check: "kb_query",
+      severity: sev,
+      summary: `scrum-master review for \`${file}\` — accepted on attempt ${attempt} by \`${model}\`${treeSplit ? " (tree-split)" : ""}`,
+      evidence: [
+        `reviewed_at: ${r.reviewed_at ?? "?"}`,
+        `preview: ${String(r.suggestions_preview ?? "").slice(0, 300).replace(/\n/g, " ")}`,
+      ],
+    });
+  }
+  return findings;
+}

auditor/checks/static.ts

@@ -61,7 +61,13 @@ export function runStaticCheck(diff: string): Finding[] {
 
       if (!isAuditorCheckerFile) {
         for (const { re, why } of BLOCK_PATTERNS) {
-          if (re.test(added)) {
+          const m = added.match(re);
+          if (m && typeof m.index === "number") {
+            // Skip if the match sits inside a quoted string literal —
+            // this is how rubric files (tests/real-world/*, prompt
+            // templates) legitimately reference the patterns they
+            // guard against, without actually executing them.
+            if (isInsideQuotedString(added, m.index)) continue;
             findings.push({
               check: "static",
               severity: "block",
@@ -154,6 +160,25 @@ function extractNewFields(addedLines: string[]): string[] {
   return Array.from(fields);
 }
 
+// True if `pos` falls inside a double- or single-quoted string on this
+// line (backtick template literals too). Walks left→right toggling the
+// "in quote" state on each unescaped quote. Good enough for single-
+// line matches; multi-line strings aren't parsed (they're extremely
+// rare in the patterns we're blocking on, and would require a proper
+// tokenizer to handle correctly).
+function isInsideQuotedString(line: string, pos: number): boolean {
+  let inDouble = false, inSingle = false, inBacktick = false;
+  for (let i = 0; i < pos; i++) {
+    const c = line[i];
+    const esc = i > 0 && line[i - 1] === "\\";
+    if (esc) continue;
+    if (c === '"' && !inSingle && !inBacktick) inDouble = !inDouble;
+    else if (c === "'" && !inDouble && !inBacktick) inSingle = !inSingle;
+    else if (c === "`" && !inDouble && !inSingle) inBacktick = !inBacktick;
+  }
+  return inDouble || inSingle || inBacktick;
+}
+
 function escape(s: string): string {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }

auditor/claim_parser.ts

@@ -49,6 +49,25 @@ const WEAK_PATTERNS: RegExp[] = [
   /\bprobably\b/i,
 ];
 
+// Empirical claims: runtime measurements / observed outcomes that can't
+// be verified from a diff (only from the actual run that produced
+// them). Example: "6/6 iterations complete, 58 cloud calls, 306s
+// end-to-end" — true, but only the test's own summary.json can
+// confirm it. Classifying as empirical lets the inference check skip
+// diff-verification and saves the ladder for falsifiable claims.
+const EMPIRICAL_PATTERNS: RegExp[] = [
+  // Iteration / attempt counts: "6/6 iterations", "attempt 5", "accepted on attempt 3"
+  /\b\d+\s*\/\s*\d+\s+(iterations?|attempts?|cycles?|runs?|shards?)\b/i,
+  /\b(accepted|resolved|converged)\s+on\s+attempt\s+\d+\b/i,
+  // Runtime metrics: "58 cloud calls", "306s end-to-end", "245s total", "5931 chars"
+  /\b\d+\s+(cloud\s+)?calls?\b/i,
+  /\b\d+\s*(ms|s|seconds?|minutes?|m)\s+(end[- ]to[- ]end|total|elapsed|duration)\b/i,
+  /\b\d+\s+chars?\b.*\b(accepted|generated|produced)\b/i,
+  // "escalated through N tiers", "N distinct models"
+  /\bescalated\s+through\s+\d+\b/i,
+  /\b\d+\s+distinct\s+(model|tier)s?\b/i,
+];
+
 export interface ParsedClaims {
   claims: Claim[];
   commits_scanned: number;
@@ -77,8 +96,21 @@ function scanText(text: string, location_prefix: string, commit_sha: string, out
     const line = lines[i];
     if (line.length < 3) continue;
 
-    // Strong patterns first — if a line matches strong, it's strong,
-    // don't double-count as moderate.
+    // Empirical match wins over everything else — if a line ALSO
+    // contains a moderate word like "complete", we still want to
+    // classify it as empirical so the inference check doesn't ask
+    // the cloud to prove "58 cloud calls" from the diff. Order:
+    // empirical → strong → moderate → weak.
+    const empirical = firstMatch(line, EMPIRICAL_PATTERNS);
+    if (empirical) {
+      out.push({
+        text: line.trim().slice(0, 200),
+        commit_sha,
+        location: `${location_prefix}:${i + 1}`,
+        strength: "empirical",
+      });
+      continue;
+    }
     const strong = firstMatch(line, STRONG_PATTERNS);
     if (strong) {
       out.push({

auditor/kb_index.ts

@@ -0,0 +1,161 @@
+// kb_index — generic on-the-fly aggregation over append-only JSONL
+// scratchpads (audit_lessons, scrum_reviews, outcomes, observer ops).
+//
+// The mem0 insight: raw rows are CHEAP and tell the full story, but
+// downstream prompts need a DEFINITION, not a log. A definition is
+// the aggregate: "this signature has fired N times across M distinct
+// scopes, first_seen=X, last_seen=Y, confidence=M/N."
+//
+// This library is the single shared aggregator. Every KB writer keeps
+// appending raw rows; every KB reader uses aggregate() instead of
+// tailing the raw stream. No second file to sync, no ADD/UPDATE/NOOP
+// routing — the stats roll up from the raw rows every time.
+//
+// Why this works past hundreds of runs:
+//   - aggregate() is bounded by distinct_signatures, not total_rows.
+//   - confidence = distinct_scopes / count — low for same-scope noise,
+//     high for cross-scope patterns. Downstream severity ramps on
+//     confidence × count, not raw count, so one unfixed PR can't
+//     inflate its own recurrence score (the classic mem0 failure).
+//   - rotation (later) moves old raw to archive files; aggregate()
+//     can still read both to compute lifetime counts when needed.
+
+import { readFile } from "node:fs/promises";
+
+export interface AggregateRow {
+  signature: string;
+  count: number;
+  distinct_scopes: number;
+  first_seen: string;
+  last_seen: string;
+  confidence: number;        // distinct_scopes / count — capped at 1.0
+  representative_summary: string;  // most-recent summary for this signature
+  scopes: string[];          // up to 20 most-recent scopes for debugging
+  checks: string[];          // distinct `check` values (audit_lessons-specific)
+}
+
+export interface AggregateOptions<T> {
+  /** How to extract the dedup key from a row. */
+  keyFn: (row: T) => string | undefined;
+  /** How to extract the "scope" — distinct scopes count gives confidence. */
+  scopeFn: (row: T) => string | undefined;
+  /** How to extract the timestamp (defaults to row.audited_at / row.reviewed_at / row.timestamp). */
+  timeFn?: (row: T) => string | undefined;
+  /** How to extract a representative summary (defaults to row.summary). */
+  summaryFn?: (row: T) => string | undefined;
+  /** Max rows to read from the JSONL tail; 0 = read all. */
+  tailLimit?: number;
+  /** Include per-row check field (for multi-check aggregates). */
+  checkFn?: (row: T) => string | undefined;
+}
+
+/**
+ * Read a JSONL file and produce the aggregate map keyed by signature.
+ * Safe on missing or malformed files — returns empty map.
+ */
+export async function aggregate<T = any>(
+  jsonlPath: string,
+  opts: AggregateOptions<T>,
+): Promise<Map<string, AggregateRow>> {
+  const out = new Map<string, AggregateRow>();
+  let raw: string;
+  try { raw = await readFile(jsonlPath, "utf8"); } catch { return out; }
+  const lines = raw.split("\n").filter(l => l.length > 0);
+  const sliceFrom = opts.tailLimit && opts.tailLimit > 0 ? Math.max(0, lines.length - opts.tailLimit) : 0;
+
+  const timeFn = opts.timeFn ?? ((r: any) => r?.audited_at ?? r?.reviewed_at ?? r?.timestamp ?? r?.ran_at);
+  const summaryFn = opts.summaryFn ?? ((r: any) => r?.summary ?? r?.representative_summary);
+
+  // Per-signature scope tracking — need counts by scope to compute
+  // distinct_scopes without double-counting a scope that appears 50
+  // times. Using a Set<scope> per signature.
+  const scopeSets = new Map<string, Set<string>>();
+  const checkSets = new Map<string, Set<string>>();
+
+  for (let i = sliceFrom; i < lines.length; i++) {
+    let row: T;
+    try { row = JSON.parse(lines[i]) as T; } catch { continue; }
+    const sig = opts.keyFn(row);
+    if (!sig) continue;
+
+    let agg = out.get(sig);
+    if (!agg) {
+      agg = {
+        signature: sig,
+        count: 0,
+        distinct_scopes: 0,
+        first_seen: "",
+        last_seen: "",
+        confidence: 0,
+        representative_summary: "",
+        scopes: [],
+        checks: [],
+      };
+      out.set(sig, agg);
+      scopeSets.set(sig, new Set<string>());
+      checkSets.set(sig, new Set<string>());
+    }
+
+    agg.count += 1;
+
+    const scope = opts.scopeFn(row);
+    if (scope !== undefined && scope !== null && scope !== "") {
+      scopeSets.get(sig)!.add(String(scope));
+      // Keep scopes array ordered by recency (newest wins — shift
+      // oldest when at cap).
+      const arr = agg.scopes;
+      const s = String(scope);
+      const existing = arr.indexOf(s);
+      if (existing >= 0) arr.splice(existing, 1);
+      arr.push(s);
+      if (arr.length > 20) arr.shift();
+    }
+
+    if (opts.checkFn) {
+      const c = opts.checkFn(row);
+      if (c) checkSets.get(sig)!.add(String(c));
+    }
+
+    const t = timeFn(row);
+    if (t) {
+      if (!agg.first_seen || t < agg.first_seen) agg.first_seen = t;
+      if (!agg.last_seen  || t > agg.last_seen)  agg.last_seen = t;
+    }
+
+    const s = summaryFn(row);
+    if (s) agg.representative_summary = String(s);
+  }
+
+  // Finalize derived fields.
+  for (const [sig, agg] of out) {
+    const scopes = scopeSets.get(sig) ?? new Set<string>();
+    agg.distinct_scopes = scopes.size;
+    agg.confidence = agg.count > 0 ? Math.min(1, agg.distinct_scopes / agg.count) : 0;
+    const checks = checkSets.get(sig);
+    if (checks) agg.checks = Array.from(checks).sort();
+  }
+  return out;
+}
+
+/**
+ * Severity policy derived from aggregate stats. The rating lives here
+ * (not in each check) so all KB readers ramp severity consistently.
+ *
+ *   - confidence × count product is the real signal.
+ *   - Low confidence (< 0.3) = same-scope noise → info regardless of count.
+ *   - Mid confidence (0.3-0.6) = mixed signal → warn at count ≥ 3.
+ *   - High confidence (> 0.6) with count ≥ 5 = block-worthy cross-cutting pattern.
+ *
+ * Callers can override by reading agg directly; this is the default
+ * policy that matches the "don't escalate one unfixed PR" discipline.
+ */
+export function ratingSeverity(agg: AggregateRow): "info" | "warn" | "block" {
+  if (agg.confidence >= 0.6 && agg.count >= 5) return "block";
+  if (agg.confidence >= 0.3 && agg.count >= 3) return "warn";
+  return "info";
+}
+
+/** Human-friendly one-line summary of an aggregate row for finding evidence. */
+export function formatAgg(agg: AggregateRow): string {
+  return `count=${agg.count} distinct_scopes=${agg.distinct_scopes} confidence=${agg.confidence.toFixed(2)} seen=[${agg.first_seen.slice(0, 10)}..${agg.last_seen.slice(0, 10)}]`;
+}

auditor/types.ts

@@ -18,7 +18,14 @@ export interface Claim {
   // Heuristic rating of how strong the claim is. "green+tested"
   // is strong; "should work" is weak. Drives sensitivity — stronger
   // claims get harder-blocked on weak evidence.
-  strength: "weak" | "moderate" | "strong";
+  //
+  // "empirical" is a separate class: runtime measurements like
+  // "N cloud calls" / "306s end-to-end" / "accepted on attempt N".
+  // These cannot be verified from a static diff — only from the test
+  // output that produced them. Inference skips diff-verification for
+  // empirical claims; they become info-level context unless a future
+  // runtime_evidence check contradicts them.
+  strength: "weak" | "moderate" | "strong" | "empirical";
 }
 
 export interface Finding {