Phase C — local-Ollama LLM review wired end-to-end
Implements PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2:
- internal/llm/ollama.go — real Ollama provider:
- HealthCheck probes /api/tags + a 1-token completion + a JSON-mode
probe ({"ok": true} round-trip), populating the model-doctor.json
schema documented in docs/LOCAL_MODEL_SETUP.md
- Complete + CompleteJSON via /api/chat with stream=false
- think=false set for ALL completions (qwen3.5:latest is reasoning-
capable but the inner-loop hot path wants direct answers, not
reasoning traces consuming the token budget — same finding as
the Lakehouse-Go chatd 2026-04-30 wave)
- internal/llm/review.go — Reviewer wrapper:
- 2-attempt flow: prompt → parse → repair-prompt → parse
- Strict JSON shape enforced; markdown fences stripped before parse
- Severity normalized to enum; out-of-range confidence clamped
- Per-file chunking (file-level for v0; function-level Phase D+)
- Bounded by review-profile max_file_bytes + max_llm_chunk_chars
- pipeline.go — Phase 2 wired between static scan + report gen:
- --enable-llm flag opts in (off by default — static-only is
cheaper and faster)
- Raw output ALWAYS saved to llm-findings.raw.json (forensics)
- Normalized findings → llm-findings.normalized.json
- LLM findings merged into the report findings list (sourced
"llm" so consumers can filter)
- Receipts honestly mark phase status: "ok" | "degraded" | "skipped"
- cli model doctor — real probes replace the Phase A stub.
Verified:
- model doctor: status="ok" with qwen3.5:latest + qwen3:latest both
loaded, basic_prompt_ok=true, json_mode_ok=true
- insecure-repo with --enable-llm: 9 LLM findings; qwen3.5 correctly
flagged SQLi, RCE, hardcoded credentials as critical with verbatim
evidence; 27s wall for 3 chunks
- clean-repo with --enable-llm: 0 LLM findings, 4 parsed chunks, 2.8s
- self-review with --enable-llm: 77 LLM findings + 83 static; 3 of
~30 chunks needed retry (PROMPT.md, REPORT_SCHEMA.md,
SCRUM_TEST_TEMPLATE.md — all eventually parsed); 5min wall
go vet + go test -short clean. Fixture stray.go now `package fixture`
so go-tooling doesn't choke on the orphan.
Phase D (validator cross-check) + Phase E (memory + diff/rules
subcommands) remain.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
70d68757f7
commit
e346b54e0f
@@ -9,12 +9,15 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"local-review-harness/internal/analyzers"
|
||||
"local-review-harness/internal/config"
|
||||
"local-review-harness/internal/git"
|
||||
"local-review-harness/internal/llm"
|
||||
"local-review-harness/internal/reporters"
|
||||
"local-review-harness/internal/scanner"
|
||||
)
|
||||
@@ -26,6 +29,7 @@ type Inputs struct {
|
||||
ModelProfile config.ModelProfile
|
||||
OutputDir string
|
||||
EmitScrum bool // true → also emit scrum-test/risk-register/sprint-backlog/acceptance-gates markdown
|
||||
EnableLLM bool // Phase C: actually call the model. Off by default — operators opt in.
|
||||
}
|
||||
|
||||
// Result is what the CLI shows the operator.
|
||||
@@ -100,15 +104,44 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) {
|
||||
receipt.Phases = append(receipt.Phases, staticPhase)
|
||||
res.OutputFiles = append(res.OutputFiles, "static-findings.json")
|
||||
|
||||
// --- Phase 2: LLM review (Phase C — not implemented in MVP) ---
|
||||
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
|
||||
Name: "llm_review", Status: "degraded",
|
||||
Errors: []string{"Phase C not implemented in MVP — see PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2"},
|
||||
})
|
||||
if res.ExitCode == 0 {
|
||||
// --- Phase 2: LLM review (Phase C) ---
|
||||
llmDegraded := true
|
||||
llmPhase := reporters.PhaseReceipt{Name: "llm_review", Status: "skipped"}
|
||||
if !in.EnableLLM {
|
||||
llmPhase.Errors = append(llmPhase.Errors, "LLM review not requested (pass --enable-llm to opt in)")
|
||||
} else {
|
||||
llmFindings, raw, llmErr := runLLMReview(ctx, scan, in)
|
||||
// Always save raw output, even on failure — operator forensics.
|
||||
rawPath := filepath.Join(in.OutputDir, "llm-findings.raw.json")
|
||||
if _, err := reporters.WriteJSON(rawPath, raw); err == nil {
|
||||
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.raw.json")
|
||||
}
|
||||
if llmErr != nil {
|
||||
llmPhase.Status = "degraded"
|
||||
llmPhase.Errors = append(llmPhase.Errors, llmErr.Error())
|
||||
} else {
|
||||
normalized := reporters.StaticFindings{
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Findings: llmFindings,
|
||||
Summary: reporters.SummarizeFindings(llmFindings),
|
||||
}
|
||||
if sha, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "llm-findings.normalized.json"), normalized); err == nil {
|
||||
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.normalized.json")
|
||||
llmPhase.OutputHash = sha
|
||||
llmPhase.Status = "ok"
|
||||
llmDegraded = false
|
||||
findings = append(findings, llmFindings...)
|
||||
res.OutputFiles = append(res.OutputFiles, "llm-findings.raw.json", "llm-findings.normalized.json")
|
||||
} else {
|
||||
llmPhase.Status = "failed"
|
||||
llmPhase.Errors = append(llmPhase.Errors, "write normalized: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
if llmDegraded && res.ExitCode == 0 {
|
||||
res.ExitCode = 66
|
||||
}
|
||||
llmDegraded := true
|
||||
receipt.Phases = append(receipt.Phases, llmPhase)
|
||||
|
||||
// --- Phase 3: validation (Phase D — also deferred) ---
|
||||
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
|
||||
@@ -175,6 +208,50 @@ func writeReceipt(outputDir string, r *reporters.Receipt, startedAt time.Time, _
|
||||
return err
|
||||
}
|
||||
|
||||
// runLLMReview chunks the scan into per-file inputs, calls the
|
||||
// reviewer, and aggregates parsed findings + raw outputs. Returns
|
||||
// (findings, raw-outputs-array-for-receipts, error). The error is
|
||||
// non-nil only when the provider is fundamentally unreachable;
|
||||
// per-chunk parse failures land as ReviewOutput.Error and don't
|
||||
// fail the whole phase.
|
||||
func runLLMReview(ctx context.Context, scan *scanner.Result, in Inputs) ([]analyzers.Finding, []llm.ReviewOutput, error) {
|
||||
prov := llm.NewOllama(in.ModelProfile.BaseURL, time.Duration(in.ModelProfile.TimeoutSeconds)*time.Second)
|
||||
hctx, hcancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer hcancel()
|
||||
hs := prov.HealthCheck(hctx, in.ModelProfile.Model, in.ModelProfile.FallbackModel)
|
||||
if !hs.ServerAvailable {
|
||||
return nil, nil, fmt.Errorf("ollama unreachable at %s — Phase 2 cannot run", in.ModelProfile.BaseURL)
|
||||
}
|
||||
if !hs.PrimaryModelAvailable && !hs.FallbackModelAvailable {
|
||||
return nil, nil, fmt.Errorf("neither primary %q nor fallback %q loaded in Ollama", in.ModelProfile.Model, in.ModelProfile.FallbackModel)
|
||||
}
|
||||
model := in.ModelProfile.Model
|
||||
if !hs.PrimaryModelAvailable {
|
||||
model = in.ModelProfile.FallbackModel
|
||||
}
|
||||
|
||||
r := llm.NewReviewer(prov, model, llm.CompleteOptions{
|
||||
Temperature: in.ModelProfile.Temperature,
|
||||
MaxTokens: 0, // let model decide
|
||||
TimeoutSeconds: in.ModelProfile.TimeoutSeconds,
|
||||
})
|
||||
|
||||
chunks := llm.ChunkInputsFromScan(scan, in.ReviewProfile.Limits.MaxFileBytes, in.ReviewProfile.Limits.MaxLLMChunkChars, func(abs string) string {
|
||||
b, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
})
|
||||
|
||||
outputs := r.ReviewBatch(ctx, chunks)
|
||||
findings := []analyzers.Finding{}
|
||||
for _, o := range outputs {
|
||||
findings = append(findings, o.Findings...)
|
||||
}
|
||||
return findings, outputs, nil
|
||||
}
|
||||
|
||||
func newRunID(t time.Time) string {
|
||||
var rb [4]byte
|
||||
_, _ = rand.Read(rb[:])
|
||||
|
||||
Reference in New Issue
Block a user