profit/matrix-agent-validated
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

checkpoint: matrix-agent-validated (2026-04-25)

Browse Source 
Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.

WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.

WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
  briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
    * UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
    * REVISE: chains versions, parent.superseded_at + superseded_by stamped
    * RETIRE: marks specific trace retired with reason, excluded from retrieval
    * HISTORY: walks chain root→tip, cycle-safe

KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces

Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
profit
2026-04-25 19:43:27 -05:00
co-authored by Claude Opus 4.7
commit ac01fffd9a
325 changed files with 82720 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
auditor/checks/dynamic.ts
+91
View File
@@ -0,0 +1,91 @@
// Dynamic execution check — runs the hybrid fixture and maps its
// layer results to auditor Findings.
//
// A layer that fails with a "not implemented / 404 / slice N" error
// gets severity=info (honest placeholder signal). A layer that fails
// any other way gets severity=warn (something actually broke).
// An info-level summary finding is always emitted carrying the real
// numbers — shipped/placeholder phase counts, per-layer latency.
import { runHybridFixture } from "../fixtures/hybrid_38_40_45.ts";
import type { Finding } from "../types.ts";
const PLACEHOLDER_MARKERS = [
"unimplemented",
" 404 ", "(404)", " 405 ", "(405)",
"slice 3", "slice 4", "slice 5",
"endpoint not built", "not yet",
];
function isPlaceholderFailure(err?: string): boolean {
if (!err) return false;
const low = err.toLowerCase();
return PLACEHOLDER_MARKERS.some(m => low.includes(m.toLowerCase()));
}
export async function runDynamicCheck(): Promise<Finding[]> {
const findings: Finding[] = [];
let result;
try {
result = await runHybridFixture();
} catch (e) {
// Fixture itself crashed — can't run dynamic check at all.
return [
{
check: "dynamic",
severity: "warn",
summary: `hybrid fixture crashed before completing: ${(e as Error).message.slice(0, 140)}`,
evidence: [(e as Error).message],
},
];
}
// Per-layer findings for every non-ok layer.
for (const layer of result.layers) {
if (layer.ok) continue;
const placeholder = isPlaceholderFailure(layer.error);
findings.push({
check: "dynamic",
severity: placeholder ? "info" : "warn",
summary: placeholder
? `hybrid fixture layer ${layer.layer} (Phase ${layer.phase}) honestly reports unimplemented`
: `hybrid fixture layer ${layer.layer} (Phase ${layer.phase}) failed — not a placeholder, a real failure`,
evidence: [
`evidence: ${layer.evidence.slice(0, 160)}`,
...(layer.error ? [`error: ${layer.error.slice(0, 160)}`] : []),
`latency_ms: ${layer.latency_ms}`,
],
});
}
// One overall summary with real numbers so the report shows what
// DID pass plus per-layer timing.
const metrics_preview = Object.entries(result.real_numbers)
.slice(0, 10)
.map(([k, v]) => `${k}=${v}`);
findings.push({
check: "dynamic",
severity: "info",
summary: `hybrid fixture overall=${result.overall}, shipped [${result.shipped_phases.join(", ")}], placeholder [${result.placeholder_phases.join(", ")}]`,
evidence: metrics_preview.length > 0 ? metrics_preview : ["no metrics emitted"],
});
// If the fixture ran at all but nothing passed, elevate one of the
// summary findings to warn — something more than "all honest
// placeholders" is wrong.
if (result.overall === "fail") {
findings.push({
check: "dynamic",
severity: "warn",
summary: `hybrid fixture: 0 layers passed (overall=fail)`,
evidence: [
"a total fixture fail usually means a precondition service is down",
"(gateway /health / sidecar / Langfuse /v1/chat) — NOT necessarily",
"the PR's code problem. Check service status before blaming the PR.",
],
});
}
return findings;
}
auditor/checks/inference.ts
+666
View File
@@ -0,0 +1,666 @@
// Cloud inference check — wraps the proven run_codereview pattern
// from llm_team_ui.py (same 3-stage framing, same cloud model) to
// critique a PR's claims against its diff.
//
// Proved out 2026-04-22 via /tmp/codereview_runner.py — gpt-oss:120b
// caught a real ternary bug in auditor/fixtures/hybrid_38_40_45.ts
// that unit tests missed. This module reuses the reviewer prompt
// shape (bugs / security / performance / style / edge cases) and
// adds claim-vs-diff specific framing.
//
// Call surface: runInferenceCheck(claims, diff) → Finding[].
// Cloud latency budget: ~60s (gpt-oss:120b reviewer typically 35-50s
// with a 15KB diff + claim list).
import type { Claim, Finding } from "../types.ts";
import { Glob } from "bun";
import { readFile, mkdir, appendFile } from "node:fs/promises";
import { extractFacts } from "../fact_extractor.ts";
const GATEWAY = process.env.LH_GATEWAY_URL ?? "http://localhost:3100";
const MODEL = process.env.LH_AUDITOR_REVIEW_MODEL ?? "gpt-oss:120b";
// Tie-breaker for claims where the N=3 consensus produces a 1-1-1
// split (genuinely borderline). Different architecture from the
// primary reviewer (gpt-oss) so the tie-break isn't correlated with
// the original disagreement. qwen3-coder:480b is a newer coding
// specialist at 480B params, well-suited to PR-diff claim verification
// and distinct in training lineage from gpt-oss.
const TIEBREAKER_MODEL = process.env.LH_AUDITOR_TIEBREAKER_MODEL ?? "qwen3-coder:480b";
const N_CONSENSUS = Number(process.env.LH_AUDITOR_CONSENSUS_N ?? 3);
const AUDIT_DISCREPANCIES_JSONL = "/home/profit/lakehouse/data/_kb/audit_discrepancies.jsonl";
// 40KB comfortably fits gpt-oss:120b's context. PR #1 (~39KB) was
// previously truncated at 15KB causing the reviewer to miss later
// files (gitea.ts, policy.ts) and flag "no Gitea client present" as a
// block finding when the file was simply outside the truncation window.
//
// Above this threshold we curate via tree-split rather than truncate,
// following the scrum_master pattern: shard the diff, summarize each
// shard against the claim-verification task, merge into a compact
// scratchpad, then ask the cloud to verify claims against the
// scratchpad. This gives the cloud full-PR fidelity without bursting
// its context window (observed failure mode: empty response or
// unparseable output when prompt exceeds model's comfortable range).
const MAX_DIFF_CHARS = 40000;
// Tree-split kicks in above this. 30KB is below MAX_DIFF_CHARS so we
// curate BEFORE truncation would happen — never lose signal to a hard
// cut. Shard size is chosen so ~10 shards cover PR #8-size diffs in a
// reasonable round-trip budget.
const CURATION_THRESHOLD = 30000;
const DIFF_SHARD_SIZE = 4500;
const CALL_TIMEOUT_MS = 120_000;
const REPO_ROOT = "/home/profit/lakehouse";
export interface InferenceContext {
pr_number: number;
head_sha: string;
}
const AUDIT_FACTS_JSONL = "/home/profit/lakehouse/data/_kb/audit_facts.jsonl";
export async function runInferenceCheck(
claims: Claim[],
diff: string,
ctx?: InferenceContext,
): Promise<Finding[]> {
if (claims.length === 0) {
return [{
check: "inference",
severity: "info",
summary: "no ship-claims extracted — skipping cloud inference",
evidence: ["parser returned empty claim list; nothing to verify against cloud"],
}];
}
// Empirical claims (runtime metrics / observed outcomes) can't be
// verified from the diff. Drop them from the cloud prompt so the
// reviewer doesn't chase ghosts. A future `runtime_evidence` check
// can validate these against data/_kb/*/summary.json outputs.
const verifiable = claims.filter(c => c.strength !== "empirical");
const empiricalCount = claims.length - verifiable.length;
if (verifiable.length === 0) {
return [{
check: "inference",
severity: "info",
summary: `all ${claims.length} claims are empirical (runtime metrics) — skipping cloud inference`,
evidence: [`empirical claims can't be verified from a static diff; needs runtime-evidence check`],
}];
}
// Diff source for the cloud prompt — either the raw diff (small
// enough to fit), or a tree-split scratchpad (curation layer). We
// prefer curation to truncation: truncation silently drops files
// past the window; curation summarizes them so the cloud still sees
// what changed, just densified.
let diffForPrompt: string;
let curationNote = "";
if (diff.length > CURATION_THRESHOLD) {
const ts = await treeSplitDiff(diff, verifiable);
diffForPrompt = ts.scratchpad;
curationNote = ` (curated: ${diff.length} chars → ${ts.shards} shards → scratchpad ${ts.scratchpad.length} chars)`;
} else {
diffForPrompt = diff;
}
// Belt-and-suspenders truncation — even a tree-split scratchpad
// shouldn't exceed MAX_DIFF_CHARS in practice, but guard anyway so
// pathological inputs can't burst the prompt.
const truncated = diffForPrompt.length > MAX_DIFF_CHARS
? diffForPrompt.slice(0, MAX_DIFF_CHARS) + `\n...[${diffForPrompt.length - MAX_DIFF_CHARS} more chars truncated]`
: diffForPrompt;
// Build the reviewer prompt in the same shape as run_codereview's
// review stage (llm_team_ui.py:10950), adapted for claim verification:
// "Task: ..."
// "Code: ..."
// "Review: bugs/security/perf/style/edge. Provide corrected code."
// We add: claim list upfront + ask for structured JSON verdict.
//
// When the diff was curated (tree-split scratchpad), we add an
// explicit anti-false-positive instruction: the scratchpad is a
// distillation, not the full source, so absence-from-scratchpad is
// NOT evidence of absence-from-diff. Mirrors the fix we made in
// scrum_master's review prompt for the same class of error.
const isCurated = curationNote.length > 0;
const curationGuard = isCurated
? [
"",
"CRITICAL: the 'Diff' below is a curated multi-shard scratchpad,",
"NOT the full raw diff. The scratchpad distills each shard down",
"to facts useful for claim verification and drops the rest.",
"DO NOT flag a function/field/feature as 'missing' or 'not",
"implemented' based solely on its absence from the scratchpad —",
"absence in a distillation is NOT evidence of absence in the",
"actual diff. Only judge a claim NOT BACKED when the scratchpad",
"DIRECTLY contradicts it (e.g. scratchpad shows the function was",
"added empty, or shows the claimed code path is a stub).",
"Skip the unflagged_gaps section entirely when operating on a",
"curated scratchpad — you can't reliably detect gaps from a",
"distillation, and false positives there are worse than misses.",
].join("\n")
: "";
const systemMsg = [
"You review pull-request diffs against the author's own ship-claims.",
"For each claim, decide: is it backed by actual code in the diff, or is",
"it placeholder / aspirational / unwired?",
"",
"A claim is BACKED when the diff contains a real code path that delivers",
"the claimed behavior. A claim is NOT BACKED when:",
" - the claim asserts functionality but the diff only adds types/fields",
" with no consumer",
" - the claim mentions tests but no test function was added",
" - the claim claims integration but the integration point is a stub",
" - the diff contains unimplemented!() / todo!() / TODO comments",
" - the claim says 'works end-to-end' but the diff has no end-to-end test",
curationGuard,
"",
"Respond with strict JSON only. No prose before or after. Shape:",
"{",
' "claim_verdicts": [',
' {"claim_idx": 0, "backed": false, "evidence": "short reason"}',
" ],",
' "unflagged_gaps": [',
' {"location": "file:line", "summary": "short description"}',
" ]",
"}",
].join("\n");
const userMsg = [
`Ship-claims the author made (numbered 0..N-1):`,
verifiable.map((c, i) => ` ${i}. [${c.strength}] "${c.text}" at ${c.location}`).join("\n"),
"",
`Diff:`,
"```",
truncated,
"```",
"",
`For each numbered claim above, emit a claim_verdicts entry. For gaps the`,
`author DIDN'T claim but that look like placeholder code, emit unflagged_gaps.`,
`Strict JSON only, matching the shape described. No prose outside JSON.`,
].join("\n");
// N=3 consensus — run the primary reviewer in parallel, collect
// all three parsed responses, majority-vote per claim. Parallel
// (Promise.all) because each call is ~20-30s and they're independent;
// wall-clock stays ~same as single call, cost 3x tokens. Empirical
// justification: in 3-run determinism tests, 7/8 findings were
// stable but 1 flipped across runs — majority vote stabilizes the
// flipping class without losing the stable signal.
const primaryRuns = await Promise.all(
Array.from({ length: N_CONSENSUS }, () =>
runCloudInference(systemMsg, userMsg, MODEL)),
);
const parsedRuns = primaryRuns.filter(r => r.parsed !== null);
if (parsedRuns.length === 0) {
// All N calls failed. Surface the first-run diagnostic so the
// operator sees *why* (unreachable / non-200 / unparseable).
const first = primaryRuns[0];
return [{
check: "inference",
severity: "info",
summary: `cloud inference all ${N_CONSENSUS} consensus runs failed — ${first.error ?? "unknown"}`,
evidence: [
`first-run diagnostic: ${first.diagnostic ?? "(none)"}`,
`successful runs: 0 / ${N_CONSENSUS}`,
],
}];
}
// Aggregate votes per claim_idx.
interface Votes { trues: number; falses: number; evidences: string[] }
const votesByClaim = new Map<number, Votes>();
const unflaggedByRun: any[][] = [];
let totalTokens = 0;
for (const run of parsedRuns) {
totalTokens += run.tokens;
unflaggedByRun.push(Array.isArray(run.parsed?.unflagged_gaps) ? run.parsed.unflagged_gaps : []);
for (const v of run.parsed?.claim_verdicts ?? []) {
const idx = Number(v?.claim_idx);
if (!Number.isFinite(idx)) continue;
const rec = votesByClaim.get(idx) ?? { trues: 0, falses: 0, evidences: [] };
if (v.backed === false) {
rec.falses++;
rec.evidences.push(String(v.evidence ?? ""));
} else if (v.backed === true) {
rec.trues++;
}
votesByClaim.set(idx, rec);
}
}
const findings: Finding[] = [];
// Summary finding so the verdict layer knows the check ran.
findings.push({
check: "inference",
severity: "info",
summary: `cloud review completed (model=${MODEL}, consensus=${parsedRuns.length}/${N_CONSENSUS}, tokens=${totalTokens})${curationNote}`,
evidence: [
`claims voted: ${votesByClaim.size}`,
`parsed runs: ${parsedRuns.length} / ${N_CONSENSUS}`,
],
});
// Per-claim majority vote; tie-break if no majority.
const discrepancies: Array<{
claim_idx: number;
claim_text: string;
votes: { trues: number; falses: number };
resolution: "majority_backed" | "majority_not_backed" | "tiebreaker_backed" | "tiebreaker_not_backed" | "unresolved";
tiebreaker_model?: string;
}> = [];
for (const [idx, votes] of votesByClaim) {
const claim = verifiable[idx];
if (!claim) continue;
const totalVotes = votes.trues + votes.falses;
let notBacked: boolean | null = null;
let resolution: typeof discrepancies[number]["resolution"] = "majority_backed";
let evidenceText = "";
let tbModel: string | undefined;
if (votes.falses > votes.trues) {
notBacked = true;
resolution = "majority_not_backed";
evidenceText = votes.evidences[0] ?? "(no reason given)";
} else if (votes.trues > votes.falses) {
notBacked = false;
resolution = "majority_backed";
} else {
// Tie. Run tie-breaker with a different-architecture model.
const tb = await runCloudInference(systemMsg, userMsg, TIEBREAKER_MODEL);
if (tb.parsed) {
const tv = (tb.parsed.claim_verdicts ?? []).find((v: any) => Number(v?.claim_idx) === idx);
if (tv?.backed === false) {
notBacked = true;
resolution = "tiebreaker_not_backed";
evidenceText = `(tie-breaker ${TIEBREAKER_MODEL}) ${String(tv.evidence ?? "")}`;
tbModel = TIEBREAKER_MODEL;
} else if (tv?.backed === true) {
notBacked = false;
resolution = "tiebreaker_backed";
tbModel = TIEBREAKER_MODEL;
} else {
resolution = "unresolved";
}
} else {
resolution = "unresolved";
}
}
// Log every case where the N runs disagreed — discrepancies are
// signal, not noise. Separate from audit_lessons.jsonl because
// they're about the *auditor's* quality, not the PR's quality.
const disagreed = totalVotes >= 2 && votes.trues > 0 && votes.falses > 0;
if (disagreed || resolution.startsWith("tiebreaker") || resolution === "unresolved") {
discrepancies.push({
claim_idx: idx,
claim_text: claim.text,
votes: { trues: votes.trues, falses: votes.falses },
resolution,
tiebreaker_model: tbModel,
});
}
if (notBacked === true) {
const sev: Finding["severity"] = claim.strength === "strong" ? "block"
: claim.strength === "moderate" ? "warn"
: "info";
findings.push({
check: "inference",
severity: sev,
claim_text: claim.text,
summary: `cloud: claim not backed — "${claim.text.slice(0, 100)}"`,
evidence: [
`at ${claim.location}`,
`consensus: ${votes.falses}/${totalVotes} not-backed (resolution: ${resolution})`,
`cloud reason: ${evidenceText.slice(0, 200)}`,
],
});
}
}
// Persist discrepancies so we can measure consensus drift over time.
if (discrepancies.length > 0 && ctx) {
persistDiscrepancies(ctx, discrepancies).catch(e =>
console.error(`[inference] discrepancy log failed: ${(e as Error).message}`));
}
// Use first run's parsed for downstream unflagged_gaps processing.
const parsed = parsedRuns[0].parsed;
// Route the curated scratchpad through llm_team's extract-facts
// pipeline when we have (a) a curated scratchpad (best signal about
// what the PR actually changed) and (b) PR context to scope facts.
// AWAITED (not fire-and-forget) so CLI callers like audit_one.ts
// don't exit before extraction lands; the systemd poller has plenty
// of headroom (90s cycle vs ~15s extraction). A failure inside
// extractAndPersistFacts is caught + logged but never throws.
if (isCurated && ctx && process.env.LH_AUDITOR_SKIP_EXTRACT !== "1") {
try {
await extractAndPersistFacts(diffForPrompt, ctx);
} catch (e) {
console.error(`[inference] fact extraction failed: ${(e as Error).message}`);
}
}
// Belt-and-suspenders: when operating on a curated scratchpad, drop
// the unflagged_gaps section entirely. The distillation can't
// reliably ground gap-detection, and false positives are worse than
// misses for this signal class. The systemMsg already asks the
// cloud to skip this section when curated — but the model may still
// emit it, so we filter here too.
const gapsToEmit = isCurated ? [] : (parsed.unflagged_gaps ?? []);
for (const g of gapsToEmit) {
const summary = String(g?.summary ?? "?");
const location = String(g?.location ?? "?");
// False-positive guard — when the cloud says "X not defined in this
// diff" or "missing implementation of X", the cloud may just mean
// "X is not in the added lines," not "X doesn't exist in the repo."
// Extract candidate symbol names and grep the repo. If any symbol
// is defined elsewhere, drop the finding — it's a known-symbol
// reference, not a placeholder.
if (/not\s+defined|missing\s+implementation|never\s+referenced\s+or\s+integrated/i.test(summary)) {
const symbols = extractSymbols(summary);
if (symbols.length > 0) {
const resolved = await symbolsExistInRepo(symbols);
if (resolved.length === symbols.length) {
// Every named symbol exists somewhere in the repo — silent drop.
continue;
}
if (resolved.length > 0) {
// Partially resolved — demote to info with a note.
findings.push({
check: "inference",
severity: "info",
summary: `cloud gap partially resolved by repo grep: ${summary.slice(0, 120)}`,
evidence: [
`location: ${location.slice(0, 140)}`,
`resolved via grep: ${resolved.join(",")}`,
`unresolved: ${symbols.filter(s => !resolved.includes(s)).join(",")}`,
],
});
continue;
}
}
}
findings.push({
check: "inference",
severity: "warn",
summary: `cloud-flagged gap not in any claim: ${summary.slice(0, 120)}`,
evidence: [`location: ${location.slice(0, 140)}`],
});
}
return findings;
}
// Single cloud call — the consensus loop calls this N times in
// parallel. Returns the parsed JSON shape + token usage + any error
// diagnostic. NEVER throws; the consensus aggregator handles partial
// failures by dropping non-parsed runs from the vote.
interface CloudRunResult {
parsed: any | null;
tokens: number;
error?: string; // "unreachable" | "non_200" | "unparseable"
diagnostic?: string; // first 200 chars for debugging
model: string;
}
async function runCloudInference(systemMsg: string, userMsg: string, model: string): Promise<CloudRunResult> {
let resp: Response;
try {
resp = await fetch(`${GATEWAY}/v1/chat`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
provider: "ollama_cloud",
model,
messages: [
{ role: "system", content: systemMsg },
{ role: "user", content: userMsg },
],
// temp=0 (greedy) + think=true. think=true is required for
// gpt-oss:120b — without it the model returns empty content
// on large prompts. Variance from the think trace is observed
// in practice, which is why we use N=3 consensus, not single-
// call determinism.
max_tokens: 3000,
temperature: 0,
think: true,
}),
signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
});
} catch (e) {
return { parsed: null, tokens: 0, error: "unreachable", diagnostic: (e as Error).message.slice(0, 200), model };
}
if (!resp.ok) {
return { parsed: null, tokens: 0, error: "non_200", diagnostic: `${resp.status}: ${(await resp.text()).slice(0, 160)}`, model };
}
let body: any;
try { body = await resp.json(); }
catch (e) { return { parsed: null, tokens: 0, error: "unparseable", diagnostic: (e as Error).message, model }; }
const content: string = body?.choices?.[0]?.message?.content ?? "";
const tokens: number = body?.usage?.total_tokens ?? 0;
const parsed = extractJson(content);
if (!parsed) {
return { parsed: null, tokens, error: "unparseable", diagnostic: content.slice(0, 200), model };
}
return { parsed, tokens, model };
}
async function persistDiscrepancies(ctx: InferenceContext, discrepancies: any[]): Promise<void> {
await mkdir("/home/profit/lakehouse/data/_kb", { recursive: true });
const rows = discrepancies.map(d => JSON.stringify({
pr_number: ctx.pr_number,
head_sha: ctx.head_sha,
logged_at: new Date().toISOString(),
...d,
}));
await appendFile(AUDIT_DISCREPANCIES_JSONL, rows.join("\n") + "\n");
}
// Extract structured knowledge from the curated scratchpad and append
// to data/_kb/audit_facts.jsonl — one row per extract run, keyed by
// PR number + head SHA for scope tracking. kb_query tails this next
// audit to surface recurring entities/relationships across PRs.
async function extractAndPersistFacts(scratchpad: string, ctx: InferenceContext): Promise<void> {
const ex = await extractFacts(scratchpad);
if (ex.error && ex.entities.length === 0 && ex.facts.length === 0) {
// Full failure — log but don't write an empty row.
console.error(`[inference] extractFacts skipped row: ${ex.error}`);
return;
}
const row = {
pr_number: ctx.pr_number,
head_sha: ctx.head_sha,
extracted_at: ex.extracted_at,
extractor: ex.extractor_model,
verifier: ex.verifier_model,
llm_team_run_id: ex.llm_team_run_id ?? null,
facts: ex.facts,
entities: ex.entities,
relationships: ex.relationships,
verification_preview: ex.verification.slice(0, 400),
verifier_verdicts: ex.verifier_verdicts,
facts_dropped_by_verifier: ex.facts_dropped_by_verifier ?? 0,
schema_version: 2,
source: "audit_inference",
};
await mkdir("/home/profit/lakehouse/data/_kb", { recursive: true });
await appendFile(AUDIT_FACTS_JSONL, JSON.stringify(row) + "\n");
}
// Curation via tree-split — ports the scrum_master pattern into the
// inference check. Shards the raw diff into DIFF_SHARD_SIZE chunks,
// summarizes each shard *against the claim-verification task* so the
// summary preserves exactly what the cloud needs to judge claims
// (function signatures, struct fields, deletions, new files), drops
// everything else. Merges into a compact scratchpad.
//
// Cost: N cloud calls for the shard summaries + 1 cloud call for the
// final verification = N+1 calls instead of 1. Mitigation: shards run
// serially (not parallel) to keep gateway load bounded; summary calls
// use max_tokens=400 so they're fast (~2s each on gpt-oss:120b).
//
// Determinism: each shard summary call uses temp=0 + think=true (same
// as the top-level inference call), so identical input yields
// identical scratchpad. The final verification call then sees a
// stable scratchpad, giving stable verdicts.
async function treeSplitDiff(
fullDiff: string,
claims: Claim[],
): Promise<{ scratchpad: string; shards: number }> {
const shards: Array<{ from: number; to: number; text: string }> = [];
for (let i = 0; i < fullDiff.length; i += DIFF_SHARD_SIZE) {
const end = Math.min(i + DIFF_SHARD_SIZE, fullDiff.length);
shards.push({ from: i, to: end, text: fullDiff.slice(i, end) });
}
// Curate the claim list into a short form the summary prompt can
// use to bias extraction toward relevant facts.
const claimDigest = claims.map((c, i) =>
`${i}. [${c.strength}] "${c.text.slice(0, 100)}"`
).join("\n");
let scratchpad = "";
for (const [si, shard] of shards.entries()) {
const prompt = [
`You are summarizing shard ${si + 1}/${shards.length} (chars ${shard.from}..${shard.to}) of a PR diff.`,
`The downstream task will verify these ship-claims against the full-PR summary. Extract ONLY facts that could confirm or refute these claims:`,
"",
claimDigest,
"",
"Extract: new function/method signatures, struct fields, deletions, new files, wiring (function X calls Y), absence-of-implementation markers, TODO comments on added lines.",
"Skip: comment-only edits, whitespace, import reordering, unrelated cosmetic changes.",
"",
"─────── shard diff ───────",
shard.text,
"─────── end shard ───────",
"",
"Output: up to 180 words of facts in bullet form. No prose preamble, no claim verdicts (that's for the downstream step).",
].join("\n");
const r = await callCloud(prompt, 400);
if (r.content) {
scratchpad += `\n--- shard ${si + 1} (chars ${shard.from}..${shard.to}) ---\n${r.content.trim()}\n`;
}
}
return { scratchpad: scratchpad.trim(), shards: shards.length };
}
// Minimal cloud caller used only by treeSplitDiff — same gateway +
// model as the top-level call, but think=false. Shards are small
// (≤DIFF_SHARD_SIZE ~4500 chars) and the task is pure fact
// extraction, not reasoning. think=true on the shards introduced
// variance in reasoning traces that compounded across 23 calls into
// a non-deterministic scratchpad (observed during curation
// validation: same-SHA runs produced 5/7/8 final findings).
// think=false on small prompts is stable — only breaks at the main
// call's 10K+ prompt size, which keeps think=true.
async function callCloud(prompt: string, maxTokens: number): Promise<{ content: string }> {
try {
const r = await fetch(`${GATEWAY}/v1/chat`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
provider: "ollama_cloud",
model: MODEL,
messages: [{ role: "user", content: prompt }],
max_tokens: maxTokens,
temperature: 0,
think: false,
}),
signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
});
if (!r.ok) return { content: "" };
const j: any = await r.json();
return { content: j?.choices?.[0]?.message?.content ?? "" };
} catch {
return { content: "" };
}
}
// Pull out plausible code-symbol names from a summary string.
// Matches:
// - identifier with backticks: `foo_bar`
// - identifier followed by parens: foo_bar()
// - CamelCase types
// - snake_case_functions
// Filters out common English words that could be matched accidentally.
const STOPWORDS = new Set([
"not","the","and","for","this","that","with","but","are","was","has",
"have","been","any","missing","implementation","diff","defined","never",
"referenced","integrated","flow","code","file","some","only","when",
]);
function extractSymbols(text: string): string[] {
const out = new Set<string>();
// `backticked` symbols
for (const m of text.matchAll(/`([A-Za-z_][A-Za-z0-9_]{2,})`/g)) out.add(m[1]);
// foo() or foo_bar() calls
for (const m of text.matchAll(/\b([A-Za-z_][A-Za-z0-9_]{2,})\s*\(/g)) out.add(m[1]);
// CamelCase types (3+ chars, must start with uppercase)
for (const m of text.matchAll(/\b([A-Z][A-Za-z0-9]{2,})\b/g)) out.add(m[1]);
return Array.from(out).filter(s => !STOPWORDS.has(s.toLowerCase()));
}
// Scan the repo for at least one definition of each symbol. Uses Bun's
// Glob to walk TS/Rust/Python/JS sources; ignores node_modules, data/,
// and target/. Skips files > 500KB — those are fixtures/snapshots that
// won't contain a definition line and slurping them slows the audit.
async function symbolsExistInRepo(symbols: string[]): Promise<string[]> {
const patterns = ["**/*.ts", "**/*.tsx", "**/*.rs", "**/*.py", "**/*.js"];
const skip = (p: string) => p.includes("/node_modules/") || p.startsWith("data/") || p.includes("/target/") || p.startsWith("dist/");
const MAX_FILE_BYTES = 500_000;
const { stat } = await import("node:fs/promises");
const resolved = new Set<string>();
const toFind = new Set(symbols);
for (const pat of patterns) {
if (toFind.size === 0) break;
const glob = new Glob(pat);
for await (const f of glob.scan({ cwd: REPO_ROOT, onlyFiles: true })) {
if (skip(f)) continue;
try { const s = await stat(`${REPO_ROOT}/${f}`); if (s.size > MAX_FILE_BYTES) continue; } catch { continue; }
let content: string;
try { content = await readFile(`${REPO_ROOT}/${f}`, "utf8"); } catch { continue; }
for (const sym of Array.from(toFind)) {
// Definition heuristics: `function sym`, `fn sym`, `const sym`,
// `let sym`, `def sym`, `class sym`, `struct sym`, `enum sym`,
// `trait sym`, `async function sym`, `pub (async )?fn sym`.
const re = new RegExp(
`\\b(function|async\\s+function|const|let|var|def|class|struct|enum|trait|impl|type|interface|fn|pub\\s+(async\\s+)?fn)\\s+${escapeRe(sym)}\\b`
);
if (re.test(content)) {
resolved.add(sym);
toFind.delete(sym);
if (toFind.size === 0) break;
}
}
}
}
return Array.from(resolved);
}
function escapeRe(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
// Lift the first balanced JSON object out of the response. Tolerates
// leading prose, code fences, and model reasoning preamble when the
// cloud model ignored "strict JSON only."
function extractJson(text: string): any | null {
const cleaned = text.replace(/^```(?:json)?\s*/im, "").replace(/```\s*$/im, "");
let depth = 0;
let start = -1;
for (let i = 0; i < cleaned.length; i++) {
const c = cleaned[i];
if (c === "{") {
if (depth === 0) start = i;
depth++;
} else if (c === "}") {
depth--;
if (depth === 0 && start >= 0) {
try { return JSON.parse(cleaned.slice(start, i + 1)); } catch { start = -1; }
}
}
}
return null;
}
auditor/checks/kb_query.ts
+397
View File
@@ -0,0 +1,397 @@
// Local-KB check — reads data/_kb/ + data/_observer/ + data/_bot/
// for prior evidence bearing on this PR's claims. Cheap, offline,
// no model calls. The point: if a claim like "Phase X shipped" has
// a historical record of failing on the same signature before, the
// auditor surfaces that pattern before the cloud check has to
// infer it.
//
// What this check reads (all file-backed, append-only or periodic):
// data/_kb/outcomes.jsonl — per-scenario outcomes (kb.ts)
// data/_kb/error_corrections.jsonl — fail→succeed deltas on same sig
// data/_kb/scrum_reviews.jsonl — scrum-master accepted reviews
// data/_observer/ops.jsonl — observer ring → disk stream
// data/_bot/cycles/*.json — bot cycle results
//
// Each JSONL line / per-cycle file is small; this check reads tails
// only (last N lines or last M files) to stay cheap on large corpora.
import { readFile, readdir, stat } from "node:fs/promises";
import { join } from "node:path";
import type { Claim, Finding } from "../types.ts";
import { aggregate, ratingSeverity, formatAgg } from "../kb_index.ts";
const KB_DIR = "/home/profit/lakehouse/data/_kb";
const OBSERVER_OPS = "/home/profit/lakehouse/data/_observer/ops.jsonl";
const BOT_CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
const SCRUM_REVIEWS_JSONL = "/home/profit/lakehouse/data/_kb/scrum_reviews.jsonl";
const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
const AUDIT_FACTS_JSONL = "/home/profit/lakehouse/data/_kb/audit_facts.jsonl";
const TAIL_LINES = 500;
const MAX_BOT_CYCLE_FILES = 30;
export async function runKbCheck(claims: Claim[], prFiles: string[] = []): Promise<Finding[]> {
const findings: Finding[] = [];
// 1. Recent scenario outcomes: are strong-claim-style phrases showing
// up alongside failed events? That's "we claimed it worked" +
// "it didn't" in the KB.
const scenarioFindings = await checkScenarioOutcomes(claims);
findings.push(...scenarioFindings);
// 2. Error corrections: any of the claims text overlap a
// recently-observed fail→succeed pair? If yes, add context.
const correctionFindings = await checkErrorCorrections(claims);
findings.push(...correctionFindings);
// 3. Bot cycles: any prior bot cycle ended in tests_failed or
// apply_failed on a file this PR is also touching?
const botFindings = await checkBotCycles();
findings.push(...botFindings);
// 4. Observer: count recent error events. High volume = shared
// infra problem, worth flagging (context for other findings).
const obsFindings = await checkObserverStream();
findings.push(...obsFindings);
// 5. Scrum-master reviews — surface prior accepted reviews for any
// file in this PR's diff. Cohesion plan Phase C wire: the
// auditor gets to "borrow" the scrum-master's deeper per-file
// analysis instead of re-doing that work.
if (prFiles.length > 0) {
const scrumFindings = await checkScrumReviews(prFiles);
findings.push(...scrumFindings);
}
// 6b. Audit-facts (llm_team extract pipeline output) — surface
// entities that recur across multiple PRs. These are the
// "core system entities" accumulating in the knowledge base;
// showing them as info on future audits gives reviewers
// architectural context the raw diff doesn't convey.
const factFindings = await checkAuditFacts();
findings.push(...factFindings);
// 6. Audit-lessons feedback loop — summarize the top recurring
// patterns from prior audits' block/warn findings. If the same
// pattern signature has fired 3+ times across prior audits,
// emit it as a block-severity finding so reviewers know this
// is a known-recurring class, not a one-off. Does NOT couple
// to the current audit's static/inference findings (those run
// in parallel and we can't see them here) — the amplification
// is emergent: if the current audit's finding-summary matches
// a top recurrence, the reviewer sees both.
const auditLessonFindings = await checkAuditLessons();
findings.push(...auditLessonFindings);
return findings;
}
async function tailJsonl<T = any>(path: string, n: number): Promise<T[]> {
try {
const raw = await readFile(path, "utf8");
const lines = raw.split("\n").filter(l => l.length > 0);
const slice = lines.slice(-n);
const out: T[] = [];
for (const line of slice) {
try { out.push(JSON.parse(line)); } catch { /* skip malformed */ }
}
return out;
} catch {
return [];
}
}
async function checkScenarioOutcomes(_claims: Claim[]): Promise<Finding[]> {
const outcomes = await tailJsonl<any>(join(KB_DIR, "outcomes.jsonl"), TAIL_LINES);
if (outcomes.length === 0) return [];
const totalEvents = outcomes.reduce((s, o) => s + (o.total_events ?? 0), 0);
const okEvents = outcomes.reduce((s, o) => s + (o.ok_events ?? 0), 0);
const failRate = totalEvents > 0 ? 1 - okEvents / totalEvents : 0;
if (totalEvents === 0) {
return [{
check: "kb_query",
severity: "info",
summary: `KB: no scenario outcomes on file — learning loop is empty`,
evidence: [`data/_kb/outcomes.jsonl has ${outcomes.length} entries with 0 total events`],
}];
}
const recent = outcomes.slice(-10);
const recentFailSigs: string[] = recent
.filter(o => (o.ok_events ?? 0) < (o.total_events ?? 0))
.map(o => o.sig_hash)
.filter(s => typeof s === "string");
const findings: Finding[] = [{
check: "kb_query",
severity: failRate > 0.3 ? "warn" : "info",
summary: `KB: ${outcomes.length} recent scenario runs, ${okEvents}/${totalEvents} events ok (fail rate ${(failRate * 100).toFixed(1)}%)`,
evidence: [
`most recent: ${recent[recent.length - 1]?.run_id ?? "?"}`,
`recent failing sigs: ${recentFailSigs.length > 0 ? recentFailSigs.slice(-3).join(", ") : "none"}`,
],
}];
return findings;
}
async function checkErrorCorrections(_claims: Claim[]): Promise<Finding[]> {
const corrections = await tailJsonl<any>(join(KB_DIR, "error_corrections.jsonl"), TAIL_LINES);
if (corrections.length === 0) return [];
return [{
check: "kb_query",
severity: "info",
summary: `KB: ${corrections.length} error corrections on file (fail→succeed pairs)`,
evidence: [
corrections.length > 0
? `most recent: ${String(corrections[corrections.length - 1]?.sig_hash ?? "?").slice(0, 24)}`
: "none",
],
}];
}
async function checkBotCycles(): Promise<Finding[]> {
let entries: string[] = [];
try { entries = await readdir(BOT_CYCLES_DIR); }
catch { return []; }
const jsonFiles = entries.filter(e => e.endsWith(".json"));
if (jsonFiles.length === 0) return [];
// Sort by mtime desc, take most recent N
const withStat = await Promise.all(
jsonFiles.map(async name => {
try { return { name, mtime: (await stat(join(BOT_CYCLES_DIR, name))).mtimeMs }; }
catch { return { name, mtime: 0 }; }
}),
);
const recent = withStat.sort((a, b) => b.mtime - a.mtime).slice(0, MAX_BOT_CYCLE_FILES);
const outcomes: Record<string, number> = {};
for (const { name } of recent) {
try {
const r = JSON.parse(await readFile(join(BOT_CYCLES_DIR, name), "utf8"));
const o = String(r.outcome ?? "unknown");
outcomes[o] = (outcomes[o] ?? 0) + 1;
} catch { /* skip */ }
}
const summary = Object.entries(outcomes)
.sort((a, b) => b[1] - a[1])
.map(([k, v]) => `${k}=${v}`)
.join(", ");
const failCount = (outcomes["tests_failed"] ?? 0) + (outcomes["apply_failed"] ?? 0) + (outcomes["model_failed"] ?? 0);
return [{
check: "kb_query",
severity: failCount > recent.length / 2 ? "warn" : "info",
summary: `KB: bot recorded ${recent.length} recent cycles — ${summary || "no outcomes parsed"}`,
evidence: [
`dir: ${BOT_CYCLES_DIR}`,
`fail-class total: ${failCount} / ${recent.length}`,
],
}];
}
async function checkObserverStream(): Promise<Finding[]> {
const ops = await tailJsonl<any>(OBSERVER_OPS, TAIL_LINES);
if (ops.length === 0) return [];
const failures = ops.filter(o => o.ok === false).length;
return [{
check: "kb_query",
severity: "info",
summary: `KB: observer stream ${ops.length} recent ops, ${failures} failures`,
evidence: [
`source: ${OBSERVER_OPS}`,
`by source: ${observerBySource(ops)}`,
],
}];
}
function observerBySource(ops: any[]): string {
const c: Record<string, number> = {};
for (const o of ops) {
const s = String(o.source ?? "unknown");
c[s] = (c[s] ?? 0) + 1;
}
return Object.entries(c).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(", ") || "empty";
}
// Audit-facts — reads data/_kb/audit_facts.jsonl (populated by every
// curated inference run via llm_team's extract pipeline). Each row
// has arrays: facts, entities, relationships. We explode entities and
// aggregate them across PRs using kb_index. An entity seen in 3+ PRs
// is a "core system entity" — we surface the top N as info context.
//
// Filters out short names (<3 chars, likely qwen2.5 truncation
// artifacts) and generic types ("string", "number") that would
// otherwise dominate the ranking.
const ENTITY_NAME_MIN_LEN = 3;
const GENERIC_ENTITY_NAMES = new Set([
"string", "number", "boolean", "any", "void", "unknown", "never",
"object", "array", "function", "const", "let", "var", "true", "false",
"null", "undefined", "promise", "map", "set", "record",
]);
async function checkAuditFacts(): Promise<Finding[]> {
// Read raw rows — each row has multiple entities, so we can't just
// use aggregate() directly (it's one-signature-per-row). Explode
// entities into (row, entity) pairs, then aggregate by entity name.
let raw: string;
try { raw = await (await import("node:fs/promises")).readFile(AUDIT_FACTS_JSONL, "utf8"); }
catch { return []; }
const lines = raw.split("\n").filter(l => l.length > 0);
if (lines.length === 0) return [];
interface EntityRow { entity_key: string; pr_number: number; type: string; name: string; description: string }
const entityRows: EntityRow[] = [];
for (const line of lines.slice(-TAIL_LINES * 2)) {
let row: any;
try { row = JSON.parse(line); } catch { continue; }
const prNum = Number(row?.pr_number);
if (!Number.isFinite(prNum)) continue;
for (const e of Array.isArray(row?.entities) ? row.entities : []) {
const name = String(e?.name ?? "").trim();
if (name.length < ENTITY_NAME_MIN_LEN) continue;
if (GENERIC_ENTITY_NAMES.has(name.toLowerCase())) continue;
entityRows.push({
entity_key: name.toLowerCase(),
pr_number: prNum,
type: String(e?.type ?? "?"),
name,
description: String(e?.description ?? "").slice(0, 160),
});
}
}
if (entityRows.length === 0) return [];
// Aggregate manually — one key per entity name, distinct_scopes by PR.
type Agg = { count: number; scopes: Set<number>; types: Set<string>; last_name: string; last_desc: string };
const byEntity = new Map<string, Agg>();
for (const r of entityRows) {
const a = byEntity.get(r.entity_key) ?? {
count: 0, scopes: new Set<number>(), types: new Set<string>(), last_name: "", last_desc: "",
};
a.count += 1;
a.scopes.add(r.pr_number);
a.types.add(r.type);
a.last_name = r.name;
a.last_desc = r.description;
byEntity.set(r.entity_key, a);
}
// Rank: require 2+ distinct PRs (same-PR entity-repeats don't count
// as "cross-cutting"). Take the top 5 to avoid flooding the verdict.
const ranked = Array.from(byEntity.entries())
.filter(([_, a]) => a.scopes.size >= 2)
.sort((a, b) => b[1].scopes.size - a[1].scopes.size || b[1].count - a[1].count)
.slice(0, 5);
if (ranked.length === 0) {
// Useful to know the KB is being populated — emit a single
// summary so operators see fact extraction is alive.
return [{
check: "kb_query",
severity: "info",
summary: `audit_facts KB has ${entityRows.length} entity-observations across ${new Set(entityRows.map(r => r.pr_number)).size} PRs (no cross-PR recurrences yet)`,
evidence: [`source: ${AUDIT_FACTS_JSONL}`],
}];
}
return ranked.map(([_, a]) => ({
check: "kb_query" as const,
severity: "info" as const,
summary: `core entity \`${a.last_name}\` recurs in ${a.scopes.size} PRs (types: ${Array.from(a.types).join(",")})`,
evidence: [
`count=${a.count} distinct_PRs=${a.scopes.size}`,
`description: ${a.last_desc.slice(0, 200)}`,
`PRs: ${Array.from(a.scopes).sort((x, y) => x - y).join(",")}`,
],
}));
}
// Audit-lessons — reads data/_kb/audit_lessons.jsonl (populated by
// every audit's appendAuditLessons). Uses the shared kb_index
// aggregator: groups by `signature`, distinct-scopes keyed by PR
// number, severity from ratingSeverity(agg) which applies the
// confidence × count rating (see kb_index.ts). This is the same
// aggregation any other KB reader uses — shared discipline, not
// per-check custom logic.
async function checkAuditLessons(): Promise<Finding[]> {
const bySig = await aggregate<any>(AUDIT_LESSONS_JSONL, {
keyFn: (r) => r?.signature,
scopeFn: (r) => (r?.pr_number !== undefined ? `pr-${r.pr_number}` : undefined),
checkFn: (r) => r?.check,
tailLimit: TAIL_LINES * 4,
});
if (bySig.size === 0) return [];
const findings: Finding[] = [];
for (const [sig, agg] of bySig) {
// Silent on first-ever occurrence — not yet signal.
if (agg.count < 2) continue;
const sev = ratingSeverity(agg);
findings.push({
check: "kb_query",
severity: sev,
summary: `recurring audit pattern (${agg.distinct_scopes} distinct PRs, ${agg.count} flaggings, conf=${agg.confidence.toFixed(2)}): ${agg.representative_summary.slice(0, 160)}`,
evidence: [
`signature=${sig}`,
`checks: ${agg.checks.join(",")}`,
`scopes: ${agg.scopes.slice(-6).join(",")}`,
formatAgg(agg),
],
});
}
return findings;
}
// Scrum-master reviews — the scrum pipeline writes one row per
// accepted per-file review. We match reviews whose `file` matches
// any path in the PR's diff, then surface the *preview* + which
// model the escalation ladder had to reach. If the scrum-master
// needed the 123B specialist or larger to resolve a file, that's
// a meaningful signal about the code's complexity — and it's
// surfaced to the PR without the auditor having to re-run the
// escalation ladder itself.
async function checkScrumReviews(prFiles: string[]): Promise<Finding[]> {
const rows = await tailJsonl<any>(SCRUM_REVIEWS_JSONL, TAIL_LINES);
if (rows.length === 0) return [];
// Match by exact file OR filename suffix — PR files arrive as
// `auditor/audit.ts`-style relative paths; scrum stores the same.
const norm = (p: string) => p.replace(/^\/+/, "").replace(/^home\/profit\/lakehouse\//, "");
const prSet = new Set(prFiles.map(norm));
// Keep only the most recent review per file (last-wins).
const latestByFile = new Map<string, any>();
for (const r of rows) {
const f = norm(String(r.file ?? ""));
if (!f) continue;
if (!prSet.has(f)) continue;
latestByFile.set(f, r);
}
if (latestByFile.size === 0) return [];
const findings: Finding[] = [];
for (const [file, r] of latestByFile) {
const model = String(r.accepted_model ?? "?");
const attempt = r.accepted_on_attempt ?? "?";
const treeSplit = !!r.tree_split_fired;
// Heuristic: if the scrum-master had to escalate past attempt 3,
// or had to tree-split, that's context the PR reviewer should see.
// Severity: info for low-escalation, warn if escalated far up
// the ladder (cloud specialist required).
const heavyEscalation = Number(attempt) >= 4;
const sev: "warn" | "info" = heavyEscalation ? "warn" : "info";
findings.push({
check: "kb_query",
severity: sev,
summary: `scrum-master review for \`${file}\` — accepted on attempt ${attempt} by \`${model}\`${treeSplit ? " (tree-split)" : ""}`,
evidence: [
`reviewed_at: ${r.reviewed_at ?? "?"}`,
`preview: ${String(r.suggestions_preview ?? "").slice(0, 300).replace(/\n/g, " ")}`,
],
});
}
return findings;
}
auditor/checks/static.ts
+184
View File
@@ -0,0 +1,184 @@
// Static diff check — grep-style, no AST, no LLM. Looks for patterns
// that are high-signal evidence of placeholder code.
//
// Findings are severity-graded:
// block — explicit non-impl markers (unimplemented!, todo!,
// panic!("not implemented"), throw new Error("not implemented"))
// warn — TODO / FIXME / XXX / HACK comments on added lines,
// new struct fields with no read-site anywhere in the diff,
// suspiciously-empty function bodies ({ Ok(()) } / {} when
// the commit message claims the fn "implements" something)
// info — hardcoded "test" / "dummy" / "placeholder" strings in
// added lines (could be real, just flag for inspection)
//
// Consumes: raw unified diff text from Gitea.
import type { Finding } from "../types.ts";
// Rust + TypeScript patterns that almost always indicate "this is
// not actually implemented yet."
const BLOCK_PATTERNS: Array<{ re: RegExp; why: string }> = [
{ re: /\bunimplemented!\s*\(/, why: "unimplemented!() macro call" },
{ re: /\btodo!\s*\(/, why: "todo!() macro call" },
{ re: /panic!\s*\(\s*"(?:not implemented|TODO|not yet|unimpl)/i, why: "panic! with not-implemented message" },
{ re: /throw\s+new\s+Error\s*\(\s*['"](?:not implemented|TODO|unimpl)/i, why: "throw Error 'not implemented'" },
];
const WARN_COMMENT_PATTERNS: Array<{ re: RegExp; why: string }> = [
{ re: /^\+.*\/\/\s*(TODO|FIXME|XXX|HACK)\b/i, why: "TODO/FIXME/XXX/HACK comment added" },
{ re: /^\+.*#\s*(TODO|FIXME|XXX|HACK)\b/i, why: "TODO/FIXME/XXX/HACK comment added" },
];
const INFO_HARDCODED_PATTERNS: Array<{ re: RegExp; why: string }> = [
{ re: /"(?:placeholder|dummy|foobar|xxx|replaceme|changeme)"/i, why: "suspicious hardcoded string" },
];
export function runStaticCheck(diff: string): Finding[] {
const findings: Finding[] = [];
// Per-file walk: only look at ADDED lines (prefix '+' but not '+++'
// which is the diff header).
const perFile = splitDiffByFile(diff);
for (const [path, lines] of perFile) {
// Skip diff bookkeeping + pure-delete files
if (!lines.some(l => l.startsWith("+") && !l.startsWith("+++"))) continue;
// The auditor's own check files literally contain the BLOCK
// patterns as regex definitions (BLOCK_PATTERNS in this file,
// prompt examples in inference.ts). Skipping BLOCK scan on these
// specific paths prevents the checker from self-flagging its own
// string literals. WARN/INFO patterns still run — those genuinely
// could indicate problems in the checker's own code (TODO
// comments don't self-define).
const isAuditorCheckerFile = path.startsWith("auditor/checks/") ||
path.startsWith("auditor/fixtures/");
for (let idx = 0; idx < lines.length; idx++) {
const line = lines[idx];
if (!line.startsWith("+") || line.startsWith("+++")) continue;
const added = line.slice(1);
if (!isAuditorCheckerFile) {
for (const { re, why } of BLOCK_PATTERNS) {
const m = added.match(re);
if (m && typeof m.index === "number") {
// Skip if the match sits inside a quoted string literal —
// this is how rubric files (tests/real-world/*, prompt
// templates) legitimately reference the patterns they
// guard against, without actually executing them.
if (isInsideQuotedString(added, m.index)) continue;
findings.push({
check: "static",
severity: "block",
summary: `${why} in ${path}`,
evidence: [`${path}:+${idx + 1}: ${added.trim().slice(0, 160)}`],
});
}
}
}
for (const { re, why } of WARN_COMMENT_PATTERNS) {
if (re.test(line)) {
findings.push({
check: "static",
severity: "warn",
summary: `${why} in ${path}`,
evidence: [`${path}:+${idx + 1}: ${added.trim().slice(0, 160)}`],
});
}
}
for (const { re, why } of INFO_HARDCODED_PATTERNS) {
if (re.test(added)) {
findings.push({
check: "static",
severity: "info",
summary: `${why} in ${path}`,
evidence: [`${path}:+${idx + 1}: ${added.trim().slice(0, 160)}`],
});
}
}
}
// "Field added but never read" heuristic — catches exactly the
// Phase 45 DocRef placeholder pattern. Limited to the diff itself:
// we're not doing a full-codebase grep here (too noisy; callers
// elsewhere might exist). The point is: if NEITHER this diff nor
// any other line in the diff reads the field, the PR is shipping
// state without a consumer.
const addedLines = lines.filter(l => l.startsWith("+") && !l.startsWith("+++"))
.map(l => l.slice(1));
const newFields = extractNewFields(addedLines);
for (const field of newFields) {
const readPattern = new RegExp(`[\\.:]\\s*${escape(field)}\\b|\\b${escape(field)}\\s*:`);
// The definition line itself matches readPattern — filter it out
// by requiring at least TWO lines in the diff mention the field
// (one defines, one reads).
const hits = addedLines.filter(l => readPattern.test(l));
if (hits.length < 2) {
findings.push({
check: "static",
severity: "warn",
summary: `field '${field}' added in ${path} but no read-site in the diff — could be placeholder state without a consumer`,
evidence: [`${path}: added '${field}' with no reader; rest of diff has ${hits.length - 1} mentions`],
});
}
}
}
return findings;
}
function splitDiffByFile(diff: string): Map<string, string[]> {
const out = new Map<string, string[]>();
let current: string | null = null;
let buf: string[] = [];
for (const line of diff.split(/\r?\n/)) {
const m = line.match(/^diff --git a\/(\S+) b\/(\S+)/);
if (m) {
if (current) out.set(current, buf);
current = m[2];
buf = [];
continue;
}
buf.push(line);
}
if (current) out.set(current, buf);
return out;
}
// Extract new `pub name: Type,` fields from added lines. Rust syntax.
// Narrowly-scoped: only matches at the start of a trimmed line,
// requires `pub ` prefix, ignores `pub fn` / `pub struct` / etc.
function extractNewFields(addedLines: string[]): string[] {
const fields = new Set<string>();
for (const line of addedLines) {
const t = line.trim();
// pub NAME: Type,
const m = t.match(/^pub\s+(?!fn\b|struct\b|enum\b|mod\b|use\b|trait\b|impl\b|const\b|static\b|type\b)(\w+)\s*:/);
if (m) fields.add(m[1]);
}
return Array.from(fields);
}
// True if `pos` falls inside a double- or single-quoted string on this
// line (backtick template literals too). Walks left→right toggling the
// "in quote" state on each unescaped quote. Good enough for single-
// line matches; multi-line strings aren't parsed (they're extremely
// rare in the patterns we're blocking on, and would require a proper
// tokenizer to handle correctly).
function isInsideQuotedString(line: string, pos: number): boolean {
let inDouble = false, inSingle = false, inBacktick = false;
for (let i = 0; i < pos; i++) {
const c = line[i];
const esc = i > 0 && line[i - 1] === "\\";
if (esc) continue;
if (c === '"' && !inSingle && !inBacktick) inDouble = !inDouble;
else if (c === "'" && !inDouble && !inBacktick) inSingle = !inSingle;
else if (c === "`" && !inDouble && !inSingle) inBacktick = !inBacktick;
}
return inDouble || inSingle || inBacktick;
}
function escape(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}