Phase D — validator cross-checks LLM findings + 2 close-out fixes
Implements PROMPT.md / docs/REVIEW_PIPELINE.md Phase 3:
"AI may suggest. Code validates."
internal/validators/validate.go — 3 hard checks per the
"Reject A Finding If" list:
- file does not exist (with path-traversal guard against the LLM
hallucinating ../../../etc/passwd)
- cited evidence does not appear in the file (verbatim or
trim-line-by-line — models often re-indent quotes when quoting code)
- line hint exceeds file line count
3 soft checks documented as open (claim semantics, suggested-fix
relevance, invented tests/commands — all need another LLM pass).
internal/validators/validate_test.go — 9 tests including:
- TestValidate_RejectsNonexistentFile (gate D1)
- TestValidate_RejectsEvidenceNotInFile
- TestValidate_RejectsLineHintBeyondFile
- TestValidate_AcceptsRealFinding
- TestValidate_AcceptsEvidenceWithDifferentLeadingWhitespace
- TestValidate_RejectsEmptyEvidence
- TestValidate_PassesThroughStaticFindings
- TestValidate_RejectsPathEscapingRepo (path-traversal protection)
- TestValidate_AcceptsRelativeRepoPath (the regression — see below)
Pipeline phase 3 wired between LLM review (Phase C) and report gen
(Phase 4). validated-findings.json contains the confirmed set;
rejected-findings.json contains rejects with per-finding reason +
detail. Receipt phase entry honest about output files + status.
=== Bug J caught ===
First Phase D run rejected EVERY real LLM finding as file_not_found
because the path-traversal check compared a relative joined path
(`tests/fixtures/insecure-repo/src/handler.go`) against an absolute
repoAbs (`/home/profit/share/.../insecure-repo`), so HasPrefix
always returned false. Both sides now resolved via filepath.Abs
before comparison. Regression test
TestValidate_AcceptsRelativeRepoPath locks this in — runs the
validator against a relative repo path AND a relative chdir, the
exact shape that hit the bug.
J's framing was honest: "I don't know what the problem is, but you
know what we're trying to accomplish." The fix-it-yourself signal
let me trace through the rejection details + see the smoking gun
in the detail string ("escapes repo root"). Without that prompt the
9 false rejections might have looked like real LLM bugs.
=== 2 close-out fixes ===
1. .gitignore: changed `/reports/latest/` → `**/reports/latest/`
(and same for `run-*`). Phase C committed 22 generated files
from `tests/fixtures/*/reports/latest/` because the original
pattern was anchored at the harness root only. Existing tracked
files removed via git rm --cached; new pattern keeps fixture
reports out of version control going forward.
2. pipeline.cleanOutputDir: pipeline now deletes the bounded list
of known per-run files at the start of each run. Before this,
a prior run's rejected-findings.json could linger when the
current run had no rejections — confused J during the bug hunt
above. cleanOutputDir is bounded (deletes only files we emit)
so operator-owned adjacent files stay.
Verified end-to-end: insecure-repo + --enable-llm → 25 confirmed
findings (16 static + 9 LLM), 0 rejected.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
e346b54e0f
commit
4dc53c5798
@@ -20,6 +20,7 @@ import (
|
||||
"local-review-harness/internal/llm"
|
||||
"local-review-harness/internal/reporters"
|
||||
"local-review-harness/internal/scanner"
|
||||
"local-review-harness/internal/validators"
|
||||
)
|
||||
|
||||
// Inputs is the bag the CLI passes to the pipeline.
|
||||
@@ -51,6 +52,13 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) {
|
||||
StartedAt: startedAt.Format(time.RFC3339Nano),
|
||||
}
|
||||
|
||||
// Clean output dir before each run so stale files from a prior
|
||||
// run can't leak into the current report set. 2026-04-30 fix:
|
||||
// before this, a previous run's rejected-findings.json could
|
||||
// stick around when the current run had no rejections, confusing
|
||||
// operators about which data was current.
|
||||
cleanOutputDir(in.OutputDir)
|
||||
|
||||
// --- Phase 0: repo intake ---
|
||||
scan, err := scanner.Walk(in.RepoPath, true)
|
||||
scanPhase := reporters.PhaseReceipt{Name: "repo_intake", Status: "ok"}
|
||||
@@ -143,11 +151,38 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) {
|
||||
}
|
||||
receipt.Phases = append(receipt.Phases, llmPhase)
|
||||
|
||||
// --- Phase 3: validation (Phase D — also deferred) ---
|
||||
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
|
||||
Name: "validation", Status: "skipped",
|
||||
Errors: []string{"Phase D not implemented in MVP — depends on Phase C"},
|
||||
})
|
||||
// --- Phase 3: validation (Phase D) ---
|
||||
// Cross-checks every LLM-sourced finding against actual file
|
||||
// content + path-traversal protection. Static findings pass
|
||||
// through promoted-to-confirmed (their evidence is already
|
||||
// grep-truthful by construction). Rejected findings land in
|
||||
// rejected-findings.json with per-rejection reason for the
|
||||
// audit trail.
|
||||
validatePhase := reporters.PhaseReceipt{Name: "validation", Status: "ok"}
|
||||
valOut := validators.Validate(in.RepoPath, findings)
|
||||
findings = valOut.Validated // pipeline downstream only sees validated set
|
||||
|
||||
if sha, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "validated-findings.json"), reporters.StaticFindings{
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Findings: valOut.Validated,
|
||||
Summary: reporters.SummarizeFindings(valOut.Validated),
|
||||
}); err != nil {
|
||||
validatePhase.Status = "failed"
|
||||
validatePhase.Errors = append(validatePhase.Errors, "validated: "+err.Error())
|
||||
} else {
|
||||
validatePhase.OutputFiles = append(validatePhase.OutputFiles, "validated-findings.json")
|
||||
validatePhase.OutputHash = sha
|
||||
}
|
||||
if len(valOut.Rejected) > 0 {
|
||||
// Rejected file is informational, not gate-blocking — the
|
||||
// audit trail belongs in version control.
|
||||
if _, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "rejected-findings.json"), valOut); err == nil {
|
||||
validatePhase.OutputFiles = append(validatePhase.OutputFiles, "rejected-findings.json")
|
||||
res.OutputFiles = append(res.OutputFiles, "rejected-findings.json")
|
||||
}
|
||||
}
|
||||
res.OutputFiles = append(res.OutputFiles, "validated-findings.json")
|
||||
receipt.Phases = append(receipt.Phases, validatePhase)
|
||||
|
||||
// --- Phase 4: report generation (markdown) ---
|
||||
if in.EmitScrum {
|
||||
@@ -252,6 +287,34 @@ func runLLMReview(ctx context.Context, scan *scanner.Result, in Inputs) ([]analy
|
||||
return findings, outputs, nil
|
||||
}
|
||||
|
||||
// cleanOutputDir removes only the files this pipeline emits. We don't
|
||||
// nuke the dir because operators might keep adjacent files there
|
||||
// (e.g. `.gitkeep`); we delete a bounded list so prior-run artifacts
|
||||
// can't masquerade as current data, but operator-owned files stay.
|
||||
func cleanOutputDir(dir string) {
|
||||
if dir == "" {
|
||||
return
|
||||
}
|
||||
known := []string{
|
||||
"repo-intake.json",
|
||||
"static-findings.json",
|
||||
"llm-findings.raw.json",
|
||||
"llm-findings.normalized.json",
|
||||
"validated-findings.json",
|
||||
"rejected-findings.json",
|
||||
"scrum-test.md",
|
||||
"risk-register.md",
|
||||
"claim-coverage-table.md",
|
||||
"sprint-backlog.md",
|
||||
"acceptance-gates.md",
|
||||
"receipts.json",
|
||||
"model-doctor.json",
|
||||
}
|
||||
for _, name := range known {
|
||||
_ = os.Remove(filepath.Join(dir, name))
|
||||
}
|
||||
}
|
||||
|
||||
func newRunID(t time.Time) string {
|
||||
var rb [4]byte
|
||||
_, _ = rand.Read(rb[:])
|
||||
|
||||
Reference in New Issue
Block a user