Auditor self-test branch: real-world pipelines + cohesion Phase C + KB index (PR #8)

Bundles 12 commits validating the auditor + scrum_master architecture end-to-end:

- enrich_prd_pipeline / hard_task_escalation / scrum_master_pipeline stress tests
- Tree-split + scrum_reviews.jsonl + kb_query surfacing
- Verdict → audit_lessons feedback loop (closed)
- kb_index aggregator with confidence-based severity policy
- 9-run + 5-run empirical tests proved the predictive-compounding property
- Level 1 correction: temp=0 cloud inference for deterministic per-claim verdicts
- audit_one.ts dry-run CLI
- Fixes: static quoted-string guard, empirical-claim classification, symbol-resolver gate, repo-file size cap

See PR #8 for run-by-run commit history.
This commit was merged in pull request #8.
This commit is contained in:
2026-04-23 03:28:32 +00:00
parent 6d7b251607
commit 156dae6732
13 changed files with 2024 additions and 13 deletions
+53 -2
View File
@@ -12,7 +12,8 @@
// review — reviews have self-review restrictions on Gitea and the
// auditor currently uses the same PAT as the PR author).
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises";
import { createHash } from "node:crypto";
import { join } from "node:path";
import type { PrSnapshot, Verdict, Finding } from "./types.ts";
import { getPrDiff, postCommitStatus, postIssueComment } from "./gitea.ts";
@@ -24,6 +25,10 @@ import { runInferenceCheck } from "./checks/inference.ts";
import { runKbCheck } from "./checks/kb_query.ts";
const VERDICTS_DIR = "/home/profit/lakehouse/data/_auditor/verdicts";
// Playbook for audit findings — one row per block/warn finding from a
// verdict. kb_query tails this next audit and escalates recurrences.
// Structured as JSONL so it's cheap to append and cheap to tail.
const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
export interface AuditOptions {
// Skip the cloud inference call (fast path for iteration). Default false.
@@ -52,7 +57,7 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
runStaticCheck(diff),
opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff),
runKbCheck(claims),
runKbCheck(claims, pr.files.map(f => f.path)),
]);
const allFindings: Finding[] = [
@@ -72,6 +77,7 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
claims_strong: claims.filter(c => c.strength === "strong").length,
claims_moderate: claims.filter(c => c.strength === "moderate").length,
claims_weak: claims.filter(c => c.strength === "weak").length,
claims_empirical: claims.filter(c => c.strength === "empirical").length,
claims_total: claims.length,
diff_bytes: diff.length,
};
@@ -80,6 +86,15 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
await persistVerdict(verdict);
// Feedback loop — every block/warn finding becomes a row in
// audit_lessons.jsonl, dedup-keyed by (check, normalized-summary).
// The next audit's kb_query reads these and escalates recurring
// findings so we don't lose the "this pattern has been flagged
// before" signal across runs. Fire-and-forget; failure here must
// not break the audit.
appendAuditLessons(verdict).catch(e =>
console.error(`[audit] audit_lessons append failed: ${(e as Error).message}`));
if (!opts.dry_run) {
await postToGitea(verdict);
}
@@ -87,6 +102,42 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
return verdict;
}
// Normalizes a finding summary for dedup: strips path-specific tails
// ("in path/to/file.ts" → "in <file>"), line numbers, and long
// commit-hash snippets. The goal is: the SAME class of finding on
// DIFFERENT files should share a signature, so we can measure
// "this pattern keeps showing up."
function normalizedSignature(f: Finding): string {
const summary = String(f.summary)
.replace(/\bin\s+\S+\.(ts|rs|js|py|md)\b/gi, "in <file>")
.replace(/:\+?\d+\b/g, ":<line>")
.replace(/[0-9a-f]{8,}/gi, "<hash>")
.replace(/\s+/g, " ")
.trim()
.slice(0, 240);
const src = `${f.check}::${f.severity}::${summary}`;
return createHash("sha256").update(src).digest("hex").slice(0, 16);
}
async function appendAuditLessons(v: Verdict): Promise<void> {
const actionable = v.findings.filter(f => f.severity === "block" || f.severity === "warn");
if (actionable.length === 0) return;
await mkdir(join(AUDIT_LESSONS_JSONL, ".."), { recursive: true });
const rows: string[] = [];
for (const f of actionable) {
rows.push(JSON.stringify({
signature: normalizedSignature(f),
check: f.check,
severity: f.severity,
summary: f.summary,
pr_number: v.pr_number,
head_sha: v.head_sha,
audited_at: v.audited_at,
}));
}
await appendFile(AUDIT_LESSONS_JSONL, rows.join("\n") + "\n");
}
async function persistVerdict(v: Verdict): Promise<void> {
await mkdir(VERDICTS_DIR, { recursive: true });
const filename = `${v.pr_number}-${v.head_sha.slice(0, 12)}.json`;