Auditor self-test branch: real-world pipelines + cohesion Phase C + KB index (PR #8)

Bundles 12 commits validating the auditor + scrum_master architecture end-to-end:

- enrich_prd_pipeline / hard_task_escalation / scrum_master_pipeline stress tests
- Tree-split + scrum_reviews.jsonl + kb_query surfacing
- Verdict → audit_lessons feedback loop (closed)
- kb_index aggregator with confidence-based severity policy
- 9-run + 5-run empirical tests proved the predictive-compounding property
- Level 1 correction: temp=0 cloud inference for deterministic per-claim verdicts
- audit_one.ts dry-run CLI
- Fixes: static quoted-string guard, empirical-claim classification, symbol-resolver gate, repo-file size cap

See PR #8 for run-by-run commit history.
This commit was merged in pull request #8.
This commit is contained in:
2026-04-23 03:28:32 +00:00
parent 6d7b251607
commit 156dae6732
13 changed files with 2024 additions and 13 deletions
+34 -2
View File
@@ -49,6 +49,25 @@ const WEAK_PATTERNS: RegExp[] = [
/\bprobably\b/i,
];
// Empirical claims: runtime measurements / observed outcomes that can't
// be verified from a diff (only from the actual run that produced
// them). Example: "6/6 iterations complete, 58 cloud calls, 306s
// end-to-end" — true, but only the test's own summary.json can
// confirm it. Classifying as empirical lets the inference check skip
// diff-verification and saves the ladder for falsifiable claims.
const EMPIRICAL_PATTERNS: RegExp[] = [
// Iteration / attempt counts: "6/6 iterations", "attempt 5", "accepted on attempt 3"
/\b\d+\s*\/\s*\d+\s+(iterations?|attempts?|cycles?|runs?|shards?)\b/i,
/\b(accepted|resolved|converged)\s+on\s+attempt\s+\d+\b/i,
// Runtime metrics: "58 cloud calls", "306s end-to-end", "245s total", "5931 chars"
/\b\d+\s+(cloud\s+)?calls?\b/i,
/\b\d+\s*(ms|s|seconds?|minutes?|m)\s+(end[- ]to[- ]end|total|elapsed|duration)\b/i,
/\b\d+\s+chars?\b.*\b(accepted|generated|produced)\b/i,
// "escalated through N tiers", "N distinct models"
/\bescalated\s+through\s+\d+\b/i,
/\b\d+\s+distinct\s+(model|tier)s?\b/i,
];
export interface ParsedClaims {
claims: Claim[];
commits_scanned: number;
@@ -77,8 +96,21 @@ function scanText(text: string, location_prefix: string, commit_sha: string, out
const line = lines[i];
if (line.length < 3) continue;
// Strong patterns first — if a line matches strong, it's strong,
// don't double-count as moderate.
// Empirical match wins over everything else — if a line ALSO
// contains a moderate word like "complete", we still want to
// classify it as empirical so the inference check doesn't ask
// the cloud to prove "58 cloud calls" from the diff. Order:
// empirical → strong → moderate → weak.
const empirical = firstMatch(line, EMPIRICAL_PATTERNS);
if (empirical) {
out.push({
text: line.trim().slice(0, 200),
commit_sha,
location: `${location_prefix}:${i + 1}`,
strength: "empirical",
});
continue;
}
const strong = firstMatch(line, STRONG_PATTERNS);
if (strong) {
out.push({