Phase C — local-Ollama LLM review wired end-to-end

Implements PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2:
- internal/llm/ollama.go — real Ollama provider:
  - HealthCheck probes /api/tags + a 1-token completion + a JSON-mode
    probe ({"ok": true} round-trip), populating the model-doctor.json
    schema documented in docs/LOCAL_MODEL_SETUP.md
  - Complete + CompleteJSON via /api/chat with stream=false
  - think=false set for ALL completions (qwen3.5:latest is reasoning-
    capable but the inner-loop hot path wants direct answers, not
    reasoning traces consuming the token budget — same finding as
    the Lakehouse-Go chatd 2026-04-30 wave)
- internal/llm/review.go — Reviewer wrapper:
  - 2-attempt flow: prompt → parse → repair-prompt → parse
  - Strict JSON shape enforced; markdown fences stripped before parse
  - Severity normalized to enum; out-of-range confidence clamped
  - Per-file chunking (file-level for v0; function-level Phase D+)
  - Bounded by review-profile max_file_bytes + max_llm_chunk_chars
- pipeline.go — Phase 2 wired between static scan + report gen:
  - --enable-llm flag opts in (off by default — static-only is
    cheaper and faster)
  - Raw output ALWAYS saved to llm-findings.raw.json (forensics)
  - Normalized findings → llm-findings.normalized.json
  - LLM findings merged into the report findings list (sourced
    "llm" so consumers can filter)
  - Receipts honestly mark phase status: "ok" | "degraded" | "skipped"
- cli model doctor — real probes replace the Phase A stub.

Verified:
- model doctor: status="ok" with qwen3.5:latest + qwen3:latest both
  loaded, basic_prompt_ok=true, json_mode_ok=true
- insecure-repo with --enable-llm: 9 LLM findings; qwen3.5 correctly
  flagged SQLi, RCE, hardcoded credentials as critical with verbatim
  evidence; 27s wall for 3 chunks
- clean-repo with --enable-llm: 0 LLM findings, 4 parsed chunks, 2.8s
- self-review with --enable-llm: 77 LLM findings + 83 static; 3 of
  ~30 chunks needed retry (PROMPT.md, REPORT_SCHEMA.md,
  SCRUM_TEST_TEMPLATE.md — all eventually parsed); 5min wall

go vet + go test -short clean. Fixture stray.go now `package fixture`
so go-tooling doesn't choke on the orphan.

Phase D (validator cross-check) + Phase E (memory + diff/rules
subcommands) remain.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude (review-harness setup)
2026-04-30 01:13:39 -05:00
co-authored by Claude Opus 4.7
parent 70d68757f7
commit e346b54e0f
27 changed files with 1785 additions and 22 deletions
+34 -14
View File
@@ -17,17 +17,19 @@ import (
"local-review-harness/internal/llm"
)
// commonFlags wires the three flags every subcommand accepts.
// commonFlags wires the flags every subcommand accepts.
type commonFlags struct {
reviewProfilePath string
modelProfilePath string
outputDir string
enableLLM bool
}
func bindCommonFlags(fs *flag.FlagSet, cf *commonFlags) {
fs.StringVar(&cf.reviewProfilePath, "review-profile", "", "review profile YAML (defaults applied if empty)")
fs.StringVar(&cf.modelProfilePath, "model-profile", "", "model profile YAML (defaults applied if empty)")
fs.StringVar(&cf.outputDir, "output-dir", "", "override review profile output dir")
fs.BoolVar(&cf.enableLLM, "enable-llm", false, "Phase C: also run local-Ollama LLM review (default off — static-only)")
}
// resolveOutputDir picks the output dir from flag > review profile >
@@ -64,10 +66,6 @@ func writeJSON(path string, v any) error {
// nowUTC returns ISO-8601 UTC for receipt timestamps.
func nowUTC() string { return time.Now().UTC().Format(time.RFC3339Nano) }
// Stub-only sentinel — Phase C replaces this with real Ollama provider.
// Phase A keeps the pipeline runnable end-to-end with a degraded-status
// model-doctor JSON.
func nilProvider() llm.Provider { return nil }
// Repo runs Phase 0 (intake) + Phase 1 (static scan) + Phase 4
// (report gen). Phase B implements the analyzers + scanner; Phase
@@ -134,20 +132,35 @@ func ModelDoctor(args []string) int {
outDir = rp.Reports.OutputDir
}
// Phase A: stub. Phase C swaps in a real probe.
// Phase C: real Ollama probe. Provider's HealthCheck does the
// actual work; we package the result into the shape REPORT_SCHEMA.md
// documents. status="ok" iff server up + at least one named
// model loaded + basic prompt + json mode all green.
prov := llm.NewOllama(mp.BaseURL, time.Duration(mp.TimeoutSeconds)*time.Second)
hctx, cancel := context.WithTimeout(context.Background(), time.Duration(mp.TimeoutSeconds)*time.Second)
defer cancel()
hs := prov.HealthCheck(hctx, mp.Model, mp.FallbackModel)
status := "ok"
if !hs.ServerAvailable {
status = "failed"
} else if !hs.BasicPromptOK || !hs.JSONModeOK || (!hs.PrimaryModelAvailable && !hs.FallbackModelAvailable) {
status = "degraded"
}
doc := map[string]any{
"provider": mp.Provider,
"base_url": mp.BaseURL,
"primary_model": mp.Model,
"fallback_model": mp.FallbackModel,
"server_available": false,
"primary_model_available": false,
"fallback_model_available": false,
"basic_prompt_ok": false,
"json_mode_ok": false,
"server_available": hs.ServerAvailable,
"primary_model_available": hs.PrimaryModelAvailable,
"fallback_model_available": hs.FallbackModelAvailable,
"basic_prompt_ok": hs.BasicPromptOK,
"json_mode_ok": hs.JSONModeOK,
"timeout_seconds": mp.TimeoutSeconds,
"status": "degraded",
"errors": []string{"phase A stub: real Ollama probe lands in Phase C"},
"status": status,
"errors": hs.Errors,
"generated_at": nowUTC(),
}
out := filepath.Join(outDir, "model-doctor.json")
@@ -156,5 +169,12 @@ func ModelDoctor(args []string) int {
return 65
}
fmt.Println(out)
return 66 // degraded exit code
switch status {
case "ok":
return 0
case "degraded":
return 66
default:
return 65
}
}
+2
View File
@@ -37,6 +37,7 @@ func runRepo(ctx context.Context, repoPath string, cf commonFlags) int {
ModelProfile: mp,
OutputDir: outDir,
EmitScrum: false,
EnableLLM: cf.enableLLM,
})
if err != nil {
fmt.Fprintln(os.Stderr, "pipeline:", err)
@@ -71,6 +72,7 @@ func runScrum(ctx context.Context, repoPath string, cf commonFlags) int {
ModelProfile: mp,
OutputDir: outDir,
EmitScrum: true,
EnableLLM: cf.enableLLM,
})
if err != nil {
fmt.Fprintln(os.Stderr, "pipeline:", err)
+235
View File
@@ -0,0 +1,235 @@
// Ollama provider — local-first per PROMPT.md.
//
// HealthCheck: probes /api/tags (server up + model list) + a 1-token
// completion + a strict-JSON probe. Used by `model doctor`.
//
// Complete + CompleteJSON: POST /api/chat with stream=false. JSON
// mode uses Ollama's native `format: "json"` — newer Ollama versions
// also accept a JSON Schema there but format=json is the lowest-
// common-denominator that works back to 0.4.
//
// `think: false` is set for ALL completions per the Lakehouse-Go
// 2026-04-30 finding: qwen3.5:latest and qwen3:latest are reasoning-
// capable but the inner-loop hot path wants direct answers, not
// `<think>` traces consuming the token budget. Callers that NEED
// reasoning override via opts (Phase F+, not yet wired).
package llm
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// OllamaProvider is the concrete impl. Stateless; safe for concurrent
// use (the http.Client handles connection pooling).
type OllamaProvider struct {
baseURL string
httpClient *http.Client
}
// NewOllama returns a provider pointed at baseURL. Empty baseURL
// defaults to http://localhost:11434. timeout 0 → 120s (matches
// model-profile default).
func NewOllama(baseURL string, timeout time.Duration) *OllamaProvider {
if baseURL == "" {
baseURL = "http://localhost:11434"
}
if timeout == 0 {
timeout = 120 * time.Second
}
return &OllamaProvider{
baseURL: strings.TrimRight(baseURL, "/"),
httpClient: &http.Client{Timeout: timeout},
}
}
func (o *OllamaProvider) Name() string { return "ollama" }
// HealthCheck runs the 5 probes documented in REPORT_SCHEMA.md
// model-doctor.json shape:
// - server_available: GET /api/tags returns 2xx
// - primary_model_available: name appears in tag list
// - fallback_model_available: name appears in tag list
// - basic_prompt_ok: a 5-token "reply OK" round-trips
// - json_mode_ok: a JSON probe parses cleanly
//
// Errors surface in HealthStatus.Errors as human-readable strings
// (no stack trace shape — operators run this from a shell).
func (o *OllamaProvider) HealthCheck(ctx context.Context, primary, fallback string) HealthStatus {
st := HealthStatus{Errors: []string{}}
// 1. Server availability + model list
tags, err := o.listTags(ctx)
if err != nil {
st.Errors = append(st.Errors, "list models: "+err.Error())
return st
}
st.ServerAvailable = true
loaded := map[string]bool{}
for _, t := range tags {
loaded[t] = true
}
st.PrimaryModelAvailable = primary != "" && loaded[primary]
st.FallbackModelAvailable = fallback != "" && loaded[fallback]
// Pick the model we'll use for the live probes — primary if
// loaded, else fallback, else the first model Ollama has.
probeModel := ""
switch {
case st.PrimaryModelAvailable:
probeModel = primary
case st.FallbackModelAvailable:
probeModel = fallback
case len(tags) > 0:
probeModel = tags[0]
st.Errors = append(st.Errors,
fmt.Sprintf("neither primary %q nor fallback %q loaded; using %q for liveness probe", primary, fallback, probeModel))
default:
st.Errors = append(st.Errors, "no models loaded; can't run liveness probe")
return st
}
// 2. Basic completion
if got, err := o.Complete(ctx, probeModel, "Reply with the single word: OK", CompleteOptions{Temperature: 0, MaxTokens: 8, TimeoutSeconds: 30}); err != nil {
st.Errors = append(st.Errors, "basic prompt: "+err.Error())
} else if strings.TrimSpace(got) != "" {
st.BasicPromptOK = true
}
// 3. JSON-mode completion
jsonGot, err := o.CompleteJSON(ctx, probeModel, `Output exactly this JSON and nothing else: {"ok": true}`, CompleteOptions{Temperature: 0, MaxTokens: 32, TimeoutSeconds: 30})
if err != nil {
st.Errors = append(st.Errors, "json mode: "+err.Error())
} else {
var probe struct{ Ok bool }
if json.Unmarshal([]byte(jsonGot), &probe) == nil {
st.JSONModeOK = true
} else {
st.Errors = append(st.Errors, "json mode: parse failed; raw="+abbrev(jsonGot, 200))
}
}
return st
}
// Complete posts to /api/chat with stream=false. Returns just the
// assistant content; token counts not surfaced (callers that need
// them go via the chat-shape API directly, which we'll expose later).
func (o *OllamaProvider) Complete(ctx context.Context, model, prompt string, opts CompleteOptions) (string, error) {
body := o.chatBody(model, prompt, opts, false)
return o.postChat(ctx, body, opts)
}
// CompleteJSON requests Ollama's native JSON-mode constrained output.
// The `format: "json"` field forces grammar-constrained generation —
// the model can only emit valid JSON. Some models still emit garbage
// in the content field (e.g. preamble text); validation is the
// caller's job (PROMPT.md "AI may suggest. Code validates.").
func (o *OllamaProvider) CompleteJSON(ctx context.Context, model, prompt string, opts CompleteOptions) (string, error) {
body := o.chatBody(model, prompt, opts, true)
return o.postChat(ctx, body, opts)
}
func (o *OllamaProvider) chatBody(model, prompt string, opts CompleteOptions, jsonMode bool) map[string]any {
options := map[string]any{}
if opts.Temperature != 0 {
options["temperature"] = opts.Temperature
}
if opts.MaxTokens > 0 {
options["num_predict"] = opts.MaxTokens
}
body := map[string]any{
"model": model,
"messages": []map[string]any{
{"role": "user", "content": prompt},
},
"stream": false,
"think": false, // local hot path skips reasoning by default
"options": options,
}
if jsonMode {
body["format"] = "json"
}
return body
}
func (o *OllamaProvider) postChat(ctx context.Context, body map[string]any, opts CompleteOptions) (string, error) {
bs, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, "POST", o.baseURL+"/api/chat", bytes.NewReader(bs))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
cli := o.httpClient
if opts.TimeoutSeconds > 0 {
cli = &http.Client{Timeout: time.Duration(opts.TimeoutSeconds) * time.Second}
}
resp, err := cli.Do(req)
if err != nil {
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return "", fmt.Errorf("ollama timeout")
}
return "", fmt.Errorf("ollama request: %w", err)
}
defer resp.Body.Close()
rb, _ := io.ReadAll(resp.Body)
if resp.StatusCode/100 != 2 {
return "", fmt.Errorf("ollama %d: %s", resp.StatusCode, abbrev(string(rb), 200))
}
var out struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
Done bool `json:"done"`
DoneReason string `json:"done_reason"`
}
if err := json.Unmarshal(rb, &out); err != nil {
return "", fmt.Errorf("ollama decode: %w (body=%s)", err, abbrev(string(rb), 200))
}
return out.Message.Content, nil
}
// listTags hits /api/tags and returns the loaded-model name list.
func (o *OllamaProvider) listTags(ctx context.Context) ([]string, error) {
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
req, _ := http.NewRequestWithContext(cctx, "GET", o.baseURL+"/api/tags", nil)
resp, err := o.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("status %d", resp.StatusCode)
}
rb, _ := io.ReadAll(resp.Body)
var out struct {
Models []struct {
Name string `json:"name"`
} `json:"models"`
}
if err := json.Unmarshal(rb, &out); err != nil {
return nil, err
}
names := make([]string, 0, len(out.Models))
for _, m := range out.Models {
names = append(names, m.Name)
}
return names, nil
}
func abbrev(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}
+295
View File
@@ -0,0 +1,295 @@
// Phase 2 (LLM review) implementation. Sends bounded chunks of the
// repo to the local model, asks for strict JSON Findings, retries
// once on parse failure, marks the phase degraded if the second
// attempt also fails. Raw output is saved either way — operators
// can re-parse manually if the harness rejected something useful.
package llm
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"local-review-harness/internal/analyzers"
"local-review-harness/internal/scanner"
)
// ReviewInput is one bounded review request. The harness chunks the
// scan result into ReviewInputs (one per file or one per file-group)
// before calling Review.
type ReviewInput struct {
ChunkID string // stable per-chunk identifier (file path for v0)
Description string // human label (e.g. "internal/foo/bar.go")
Content string // the actual code/content to review
Language string // for the prompt context
}
// ReviewOutput is what one Review call produces. RawContent is the
// model's verbatim output before parsing — saved for forensics if
// parsing fails.
type ReviewOutput struct {
ChunkID string `json:"chunk_id"`
Findings []analyzers.Finding `json:"findings"`
RawContent string `json:"raw_content"`
Parsed bool `json:"parsed"`
Retried bool `json:"retried"`
Error string `json:"error,omitempty"`
}
// Reviewer wraps a Provider with the prompt + retry logic. Stateless;
// the prompt template is baked in for v0.
type Reviewer struct {
prov Provider
model string
opts CompleteOptions
}
// NewReviewer constructs a Reviewer pointing at the configured
// primary model. opts are passed through to every Complete call;
// callers tune via review-profile.
func NewReviewer(prov Provider, model string, opts CompleteOptions) *Reviewer {
if opts.TimeoutSeconds == 0 {
opts.TimeoutSeconds = 120
}
return &Reviewer{prov: prov, model: model, opts: opts}
}
// Review runs the 2-attempt flow: prompt → parse → retry-with-repair-prompt → parse.
func (r *Reviewer) Review(ctx context.Context, in ReviewInput) ReviewOutput {
out := ReviewOutput{ChunkID: in.ChunkID}
// Attempt 1
prompt := buildReviewPrompt(in, false)
raw, err := r.prov.CompleteJSON(ctx, r.model, prompt, r.opts)
out.RawContent = raw
if err != nil {
out.Error = "request failed: " + err.Error()
return out
}
if findings, perr := parseFindings(raw, in); perr == nil {
out.Findings = findings
out.Parsed = true
return out
}
// Attempt 2 (repair prompt — feed the raw output back + ask for
// strict JSON only). Done once; second failure is degraded.
out.Retried = true
repair := buildRepairPrompt(in, raw)
raw2, err := r.prov.CompleteJSON(ctx, r.model, repair, r.opts)
out.RawContent = raw + "\n\n---repair---\n\n" + raw2
if err != nil {
out.Error = "repair request failed: " + err.Error()
return out
}
if findings, perr := parseFindings(raw2, in); perr == nil {
out.Findings = findings
out.Parsed = true
return out
} else {
out.Error = "parse failed after repair: " + perr.Error()
}
return out
}
// ReviewBatch runs Review over a slice of inputs sequentially. Could
// parallelize at G3+, but local Ollama is GPU-bound and serial is
// the safe v0 — burst-parallel would queue at the model server anyway.
func (r *Reviewer) ReviewBatch(ctx context.Context, inputs []ReviewInput) []ReviewOutput {
out := make([]ReviewOutput, 0, len(inputs))
for _, in := range inputs {
select {
case <-ctx.Done():
out = append(out, ReviewOutput{
ChunkID: in.ChunkID,
Error: "context cancelled before chunk processed",
})
continue
default:
}
out = append(out, r.Review(ctx, in))
}
return out
}
// === prompts ===
const reviewSystemPrompt = `You are a senior code reviewer auditing a single source file.
Your job: emit a JSON object with a "findings" array. Each finding
must include:
- title (string, < 80 chars)
- severity ("low" | "medium" | "high" | "critical")
- file (string, the file path you were asked to review — verbatim)
- line_hint (string, e.g. "42" or "100-110")
- evidence (string, a SHORT direct quote from the file — must
exist verbatim in the source so a downstream validator can
grep it)
- reason (string, one sentence explaining why this is a finding)
- suggested_fix (string, optional, one sentence)
- confidence (number 0.0–1.0)
Severity guidance:
- critical: credential leak, RCE risk, destructive command,
unauthenticated mutation
- high: SQL injection, broad CORS, fail-open auth, unsafe FS
- medium: hardcoded paths, weak error handling, missing tests
near important code
- low: naming, duplication, doc drift
Hard rules (failure = your output is rejected):
1. Output ONLY the JSON object. No prose before or after.
2. The evidence field MUST be a verbatim substring of the file.
If you can't quote the source, drop the finding.
3. Don't invent file paths, line numbers, or test names.
4. If the file is clean, return {"findings": []}.
5. Output nothing else when you're done.`
func buildReviewPrompt(in ReviewInput, _ bool) string {
var b strings.Builder
b.WriteString(reviewSystemPrompt)
b.WriteString("\n\n---\n\n")
b.WriteString("File path: ")
b.WriteString(in.Description)
b.WriteString("\nLanguage: ")
b.WriteString(in.Language)
b.WriteString("\n\nFile content:\n```\n")
b.WriteString(in.Content)
b.WriteString("\n```\n\nReturn JSON only.")
return b.String()
}
func buildRepairPrompt(in ReviewInput, prev string) string {
var b strings.Builder
b.WriteString("Your previous output was not valid JSON or did not match the required schema.\n\n")
b.WriteString("Required shape:\n")
b.WriteString(`{"findings":[{"title":"...","severity":"...","file":"...","line_hint":"...","evidence":"...","reason":"...","confidence":0.0}]}`)
b.WriteString("\n\nPrevious raw output (for your reference):\n")
b.WriteString(abbrev(prev, 1500))
b.WriteString("\n\nFor reference, the file you were reviewing was:\n")
b.WriteString(in.Description)
b.WriteString("\n\nReturn ONLY the JSON object now. No explanation, no markdown fences, no apology. JSON only.")
return b.String()
}
// === parsing ===
func parseFindings(raw string, in ReviewInput) ([]analyzers.Finding, error) {
// Strip leading/trailing whitespace + common markdown fences.
cleaned := strings.TrimSpace(raw)
cleaned = strings.TrimPrefix(cleaned, "```json")
cleaned = strings.TrimPrefix(cleaned, "```")
cleaned = strings.TrimSuffix(cleaned, "```")
cleaned = strings.TrimSpace(cleaned)
if cleaned == "" {
return nil, fmt.Errorf("empty content")
}
var shell struct {
Findings []struct {
Title string `json:"title"`
Severity string `json:"severity"`
File string `json:"file"`
LineHint string `json:"line_hint"`
Evidence string `json:"evidence"`
Reason string `json:"reason"`
SuggestedFix string `json:"suggested_fix"`
Confidence float64 `json:"confidence"`
} `json:"findings"`
}
if err := json.Unmarshal([]byte(cleaned), &shell); err != nil {
return nil, fmt.Errorf("unmarshal: %w", err)
}
out := make([]analyzers.Finding, 0, len(shell.Findings))
for _, f := range shell.Findings {
sev := normalizeSeverity(f.Severity)
if sev == "" {
continue // model emitted a value we don't accept
}
// Use the chunk's file path if model omitted/lied
filePath := f.File
if filePath == "" {
filePath = in.Description
}
out = append(out, analyzers.Finding{
Title: truncate(f.Title, 80),
Severity: sev,
Status: analyzers.StatusSuspected, // validator (Phase D) promotes to confirmed
File: filePath,
LineHint: f.LineHint,
Evidence: f.Evidence,
Reason: f.Reason,
SuggestedFix: f.SuggestedFix,
Source: analyzers.SourceLLM,
Confidence: clampFloat(f.Confidence, 0, 1),
CheckID: "llm.review",
})
}
return out, nil
}
func normalizeSeverity(s string) analyzers.Severity {
switch strings.ToLower(strings.TrimSpace(s)) {
case "low":
return analyzers.SeverityLow
case "medium", "med":
return analyzers.SeverityMedium
case "high":
return analyzers.SeverityHigh
case "critical", "crit":
return analyzers.SeverityCritical
}
return ""
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n]
}
func clampFloat(v, lo, hi float64) float64 {
if v < lo {
return lo
}
if v > hi {
return hi
}
return v
}
// === chunking ===
// ChunkInputsFromScan produces one ReviewInput per file under the
// configured size limit. Files larger than maxBytes are skipped (the
// LLM phase notes them in the receipt as "skipped: too large"). v0
// is per-file; per-function chunking lands in Phase D+.
func ChunkInputsFromScan(scan *scanner.Result, maxBytes int, maxChunkChars int, readFile func(abs string) string) []ReviewInput {
out := []ReviewInput{}
for _, f := range scan.Files {
if f.Language == "" {
continue // non-code files: skip LLM review (analyzers may still flag)
}
if f.Size > int64(maxBytes) {
continue
}
content := readFile(f.Abs)
if len(content) > maxChunkChars {
content = content[:maxChunkChars] + "\n... (truncated for LLM context)\n"
}
out = append(out, ReviewInput{
ChunkID: f.Path,
Description: f.Path,
Content: content,
Language: f.Language,
})
}
return out
}
// Useful for callers wiring a deadline across the whole batch.
var _ = time.Now
+84 -7
View File
@@ -9,12 +9,15 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"time"
"local-review-harness/internal/analyzers"
"local-review-harness/internal/config"
"local-review-harness/internal/git"
"local-review-harness/internal/llm"
"local-review-harness/internal/reporters"
"local-review-harness/internal/scanner"
)
@@ -26,6 +29,7 @@ type Inputs struct {
ModelProfile config.ModelProfile
OutputDir string
EmitScrum bool // true → also emit scrum-test/risk-register/sprint-backlog/acceptance-gates markdown
EnableLLM bool // Phase C: actually call the model. Off by default — operators opt in.
}
// Result is what the CLI shows the operator.
@@ -100,15 +104,44 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) {
receipt.Phases = append(receipt.Phases, staticPhase)
res.OutputFiles = append(res.OutputFiles, "static-findings.json")
// --- Phase 2: LLM review (Phase C — not implemented in MVP) ---
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
Name: "llm_review", Status: "degraded",
Errors: []string{"Phase C not implemented in MVP — see PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2"},
})
if res.ExitCode == 0 {
// --- Phase 2: LLM review (Phase C) ---
llmDegraded := true
llmPhase := reporters.PhaseReceipt{Name: "llm_review", Status: "skipped"}
if !in.EnableLLM {
llmPhase.Errors = append(llmPhase.Errors, "LLM review not requested (pass --enable-llm to opt in)")
} else {
llmFindings, raw, llmErr := runLLMReview(ctx, scan, in)
// Always save raw output, even on failure — operator forensics.
rawPath := filepath.Join(in.OutputDir, "llm-findings.raw.json")
if _, err := reporters.WriteJSON(rawPath, raw); err == nil {
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.raw.json")
}
if llmErr != nil {
llmPhase.Status = "degraded"
llmPhase.Errors = append(llmPhase.Errors, llmErr.Error())
} else {
normalized := reporters.StaticFindings{
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
Findings: llmFindings,
Summary: reporters.SummarizeFindings(llmFindings),
}
if sha, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "llm-findings.normalized.json"), normalized); err == nil {
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.normalized.json")
llmPhase.OutputHash = sha
llmPhase.Status = "ok"
llmDegraded = false
findings = append(findings, llmFindings...)
res.OutputFiles = append(res.OutputFiles, "llm-findings.raw.json", "llm-findings.normalized.json")
} else {
llmPhase.Status = "failed"
llmPhase.Errors = append(llmPhase.Errors, "write normalized: "+err.Error())
}
}
}
if llmDegraded && res.ExitCode == 0 {
res.ExitCode = 66
}
llmDegraded := true
receipt.Phases = append(receipt.Phases, llmPhase)
// --- Phase 3: validation (Phase D — also deferred) ---
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
@@ -175,6 +208,50 @@ func writeReceipt(outputDir string, r *reporters.Receipt, startedAt time.Time, _
return err
}
// runLLMReview chunks the scan into per-file inputs, calls the
// reviewer, and aggregates parsed findings + raw outputs. Returns
// (findings, raw-outputs-array-for-receipts, error). The error is
// non-nil only when the provider is fundamentally unreachable;
// per-chunk parse failures land as ReviewOutput.Error and don't
// fail the whole phase.
func runLLMReview(ctx context.Context, scan *scanner.Result, in Inputs) ([]analyzers.Finding, []llm.ReviewOutput, error) {
prov := llm.NewOllama(in.ModelProfile.BaseURL, time.Duration(in.ModelProfile.TimeoutSeconds)*time.Second)
hctx, hcancel := context.WithTimeout(ctx, 5*time.Second)
defer hcancel()
hs := prov.HealthCheck(hctx, in.ModelProfile.Model, in.ModelProfile.FallbackModel)
if !hs.ServerAvailable {
return nil, nil, fmt.Errorf("ollama unreachable at %s — Phase 2 cannot run", in.ModelProfile.BaseURL)
}
if !hs.PrimaryModelAvailable && !hs.FallbackModelAvailable {
return nil, nil, fmt.Errorf("neither primary %q nor fallback %q loaded in Ollama", in.ModelProfile.Model, in.ModelProfile.FallbackModel)
}
model := in.ModelProfile.Model
if !hs.PrimaryModelAvailable {
model = in.ModelProfile.FallbackModel
}
r := llm.NewReviewer(prov, model, llm.CompleteOptions{
Temperature: in.ModelProfile.Temperature,
MaxTokens: 0, // let model decide
TimeoutSeconds: in.ModelProfile.TimeoutSeconds,
})
chunks := llm.ChunkInputsFromScan(scan, in.ReviewProfile.Limits.MaxFileBytes, in.ReviewProfile.Limits.MaxLLMChunkChars, func(abs string) string {
b, err := os.ReadFile(abs)
if err != nil {
return ""
}
return string(b)
})
outputs := r.ReviewBatch(ctx, chunks)
findings := []analyzers.Finding{}
for _, o := range outputs {
findings = append(findings, o.Findings...)
}
return findings, outputs, nil
}
func newRunID(t time.Time) string {
var rb [4]byte
_, _ = rand.Read(rb[:])