Phase C — local-Ollama LLM review wired end-to-end

Implements PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2:
- internal/llm/ollama.go — real Ollama provider:
  - HealthCheck probes /api/tags + a 1-token completion + a JSON-mode
    probe ({"ok": true} round-trip), populating the model-doctor.json
    schema documented in docs/LOCAL_MODEL_SETUP.md
  - Complete + CompleteJSON via /api/chat with stream=false
  - think=false set for ALL completions (qwen3.5:latest is reasoning-
    capable but the inner-loop hot path wants direct answers, not
    reasoning traces consuming the token budget — same finding as
    the Lakehouse-Go chatd 2026-04-30 wave)
- internal/llm/review.go — Reviewer wrapper:
  - 2-attempt flow: prompt → parse → repair-prompt → parse
  - Strict JSON shape enforced; markdown fences stripped before parse
  - Severity normalized to enum; out-of-range confidence clamped
  - Per-file chunking (file-level for v0; function-level Phase D+)
  - Bounded by review-profile max_file_bytes + max_llm_chunk_chars
- pipeline.go — Phase 2 wired between static scan + report gen:
  - --enable-llm flag opts in (off by default — static-only is
    cheaper and faster)
  - Raw output ALWAYS saved to llm-findings.raw.json (forensics)
  - Normalized findings → llm-findings.normalized.json
  - LLM findings merged into the report findings list (sourced
    "llm" so consumers can filter)
  - Receipts honestly mark phase status: "ok" | "degraded" | "skipped"
- cli model doctor — real probes replace the Phase A stub.

Verified:
- model doctor: status="ok" with qwen3.5:latest + qwen3:latest both
  loaded, basic_prompt_ok=true, json_mode_ok=true
- insecure-repo with --enable-llm: 9 LLM findings; qwen3.5 correctly
  flagged SQLi, RCE, hardcoded credentials as critical with verbatim
  evidence; 27s wall for 3 chunks
- clean-repo with --enable-llm: 0 LLM findings, 4 parsed chunks, 2.8s
- self-review with --enable-llm: 77 LLM findings + 83 static; 3 of
  ~30 chunks needed retry (PROMPT.md, REPORT_SCHEMA.md,
  SCRUM_TEST_TEMPLATE.md — all eventually parsed); 5min wall

go vet + go test -short clean. Fixture stray.go now `package fixture`
so go-tooling doesn't choke on the orphan.

Phase D (validator cross-check) + Phase E (memory + diff/rules
subcommands) remain.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude (review-harness setup)
2026-04-30 01:13:39 -05:00
co-authored by Claude Opus 4.7
parent 70d68757f7
commit e346b54e0f
27 changed files with 1785 additions and 22 deletions
+84 -7
View File
@@ -9,12 +9,15 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"time"
"local-review-harness/internal/analyzers"
"local-review-harness/internal/config"
"local-review-harness/internal/git"
"local-review-harness/internal/llm"
"local-review-harness/internal/reporters"
"local-review-harness/internal/scanner"
)
@@ -26,6 +29,7 @@ type Inputs struct {
ModelProfile config.ModelProfile
OutputDir string
EmitScrum bool // true → also emit scrum-test/risk-register/sprint-backlog/acceptance-gates markdown
EnableLLM bool // Phase C: actually call the model. Off by default — operators opt in.
}
// Result is what the CLI shows the operator.
@@ -100,15 +104,44 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) {
receipt.Phases = append(receipt.Phases, staticPhase)
res.OutputFiles = append(res.OutputFiles, "static-findings.json")
// --- Phase 2: LLM review (Phase C — not implemented in MVP) ---
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
Name: "llm_review", Status: "degraded",
Errors: []string{"Phase C not implemented in MVP — see PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2"},
})
if res.ExitCode == 0 {
// --- Phase 2: LLM review (Phase C) ---
llmDegraded := true
llmPhase := reporters.PhaseReceipt{Name: "llm_review", Status: "skipped"}
if !in.EnableLLM {
llmPhase.Errors = append(llmPhase.Errors, "LLM review not requested (pass --enable-llm to opt in)")
} else {
llmFindings, raw, llmErr := runLLMReview(ctx, scan, in)
// Always save raw output, even on failure — operator forensics.
rawPath := filepath.Join(in.OutputDir, "llm-findings.raw.json")
if _, err := reporters.WriteJSON(rawPath, raw); err == nil {
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.raw.json")
}
if llmErr != nil {
llmPhase.Status = "degraded"
llmPhase.Errors = append(llmPhase.Errors, llmErr.Error())
} else {
normalized := reporters.StaticFindings{
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
Findings: llmFindings,
Summary: reporters.SummarizeFindings(llmFindings),
}
if sha, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "llm-findings.normalized.json"), normalized); err == nil {
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.normalized.json")
llmPhase.OutputHash = sha
llmPhase.Status = "ok"
llmDegraded = false
findings = append(findings, llmFindings...)
res.OutputFiles = append(res.OutputFiles, "llm-findings.raw.json", "llm-findings.normalized.json")
} else {
llmPhase.Status = "failed"
llmPhase.Errors = append(llmPhase.Errors, "write normalized: "+err.Error())
}
}
}
if llmDegraded && res.ExitCode == 0 {
res.ExitCode = 66
}
llmDegraded := true
receipt.Phases = append(receipt.Phases, llmPhase)
// --- Phase 3: validation (Phase D — also deferred) ---
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
@@ -175,6 +208,50 @@ func writeReceipt(outputDir string, r *reporters.Receipt, startedAt time.Time, _
return err
}
// runLLMReview chunks the scan into per-file inputs, calls the
// reviewer, and aggregates parsed findings + raw outputs. Returns
// (findings, raw-outputs-array-for-receipts, error). The error is
// non-nil only when the provider is fundamentally unreachable;
// per-chunk parse failures land as ReviewOutput.Error and don't
// fail the whole phase.
func runLLMReview(ctx context.Context, scan *scanner.Result, in Inputs) ([]analyzers.Finding, []llm.ReviewOutput, error) {
prov := llm.NewOllama(in.ModelProfile.BaseURL, time.Duration(in.ModelProfile.TimeoutSeconds)*time.Second)
hctx, hcancel := context.WithTimeout(ctx, 5*time.Second)
defer hcancel()
hs := prov.HealthCheck(hctx, in.ModelProfile.Model, in.ModelProfile.FallbackModel)
if !hs.ServerAvailable {
return nil, nil, fmt.Errorf("ollama unreachable at %s — Phase 2 cannot run", in.ModelProfile.BaseURL)
}
if !hs.PrimaryModelAvailable && !hs.FallbackModelAvailable {
return nil, nil, fmt.Errorf("neither primary %q nor fallback %q loaded in Ollama", in.ModelProfile.Model, in.ModelProfile.FallbackModel)
}
model := in.ModelProfile.Model
if !hs.PrimaryModelAvailable {
model = in.ModelProfile.FallbackModel
}
r := llm.NewReviewer(prov, model, llm.CompleteOptions{
Temperature: in.ModelProfile.Temperature,
MaxTokens: 0, // let model decide
TimeoutSeconds: in.ModelProfile.TimeoutSeconds,
})
chunks := llm.ChunkInputsFromScan(scan, in.ReviewProfile.Limits.MaxFileBytes, in.ReviewProfile.Limits.MaxLLMChunkChars, func(abs string) string {
b, err := os.ReadFile(abs)
if err != nil {
return ""
}
return string(b)
})
outputs := r.ReviewBatch(ctx, chunks)
findings := []analyzers.Finding{}
for _, o := range outputs {
findings = append(findings, o.Findings...)
}
return findings, outputs, nil
}
func newRunID(t time.Time) string {
var rb [4]byte
_, _ = rand.Read(rb[:])