Audit pipeline PR #9: determinism + fact extraction + verifier gate + KB stats + context injection (PR #9)
Bundles PR #9's work for the audit pipeline: - N=3 consensus on cloud inference (gpt-oss:120b parallel) with qwen3-coder:480b tie-breaker - audit_discrepancies.jsonl logs N-run disagreements - scrum_master reviews route through llm_team fact extraction; source="scrum_review" - Verifier-gated persistence: drops INCORRECT, keeps UNVERIFIABLE/UNCHECKED; schema_version:2 - scrum_master_reviewed flag on accepted reviews - auditor/kb_stats.ts: on-demand observability script - claim_parser history/proof pattern class (verified-on-PR, was-flipping, the-proven-X) - claim_parser quoted-string guard (mirrors static.ts fix) - fact_extractor project context injection via docs/AUDITOR_CONTEXT.md - Fixed verifier-verdict parser to handle multiple gemma2 output formats Empirical: 3-run determinism test on unchanged PR #9 SHA showed 7/7 warn findings stable; block count oscillation eliminated; llm_team quality scores 8-9 on context-injected extract runs. See PR #9 for full run-by-run commit history.
This commit was merged in pull request #9.
This commit is contained in:
@@ -343,12 +343,50 @@ Respond with markdown. Be specific, not generic. Cite file-region + PRD-chunk-of
|
||||
attempts_made: history.length,
|
||||
tree_split_fired: treeSplitFired,
|
||||
suggestions_preview: accepted.slice(0, 2000),
|
||||
schema_version: 2,
|
||||
scrum_master_reviewed: true,
|
||||
};
|
||||
try {
|
||||
await appendFile(SCRUM_REVIEWS_JSONL, JSON.stringify(row) + "\n");
|
||||
} catch (e) {
|
||||
console.error(`[scrum] failed to append scrum_reviews.jsonl: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
// Route the accepted review through llm_team's fact extractor so
|
||||
// its entities + relationships land in audit_facts.jsonl alongside
|
||||
// inference-side extractions. Same index, two sources. Tagged
|
||||
// source:"scrum_review" + scrum_master_reviewed:true so downstream
|
||||
// queries can filter by provenance. Reviews shorter than 120
|
||||
// chars are skipped — they're usually one-liners ("LGTM") with
|
||||
// no extractable knowledge.
|
||||
if (accepted.length >= 120 && process.env.LH_SCRUM_SKIP_EXTRACT !== "1") {
|
||||
try {
|
||||
const { extractFacts } = await import("../../auditor/fact_extractor.ts");
|
||||
const ex = await extractFacts(accepted);
|
||||
if (!ex.error || ex.entities.length + ex.facts.length > 0) {
|
||||
const factRow = {
|
||||
pr_number: 0, // scrum runs outside a PR scope
|
||||
file: rel,
|
||||
head_sha: "", // no SHA scope; scope is the file+timestamp
|
||||
extracted_at: ex.extracted_at,
|
||||
extractor: ex.extractor_model,
|
||||
verifier: ex.verifier_model,
|
||||
llm_team_run_id: ex.llm_team_run_id ?? null,
|
||||
facts: ex.facts,
|
||||
entities: ex.entities,
|
||||
relationships: ex.relationships,
|
||||
verification_preview: ex.verification.slice(0, 400),
|
||||
schema_version: 2,
|
||||
source: "scrum_review",
|
||||
scrum_master_reviewed: true,
|
||||
};
|
||||
const AUDIT_FACTS_JSONL = "/home/profit/lakehouse/data/_kb/audit_facts.jsonl";
|
||||
await appendFile(AUDIT_FACTS_JSONL, JSON.stringify(factRow) + "\n");
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[scrum] fact extraction failed for ${rel}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return review;
|
||||
|
||||
Reference in New Issue
Block a user