Phase C — local-Ollama LLM review wired end-to-end
Implements PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2:
- internal/llm/ollama.go — real Ollama provider:
- HealthCheck probes /api/tags + a 1-token completion + a JSON-mode
probe ({"ok": true} round-trip), populating the model-doctor.json
schema documented in docs/LOCAL_MODEL_SETUP.md
- Complete + CompleteJSON via /api/chat with stream=false
- think=false set for ALL completions (qwen3.5:latest is reasoning-
capable but the inner-loop hot path wants direct answers, not
reasoning traces consuming the token budget — same finding as
the Lakehouse-Go chatd 2026-04-30 wave)
- internal/llm/review.go — Reviewer wrapper:
- 2-attempt flow: prompt → parse → repair-prompt → parse
- Strict JSON shape enforced; markdown fences stripped before parse
- Severity normalized to enum; out-of-range confidence clamped
- Per-file chunking (file-level for v0; function-level Phase D+)
- Bounded by review-profile max_file_bytes + max_llm_chunk_chars
- pipeline.go — Phase 2 wired between static scan + report gen:
- --enable-llm flag opts in (off by default — static-only is
cheaper and faster)
- Raw output ALWAYS saved to llm-findings.raw.json (forensics)
- Normalized findings → llm-findings.normalized.json
- LLM findings merged into the report findings list (sourced
"llm" so consumers can filter)
- Receipts honestly mark phase status: "ok" | "degraded" | "skipped"
- cli model doctor — real probes replace the Phase A stub.
Verified:
- model doctor: status="ok" with qwen3.5:latest + qwen3:latest both
loaded, basic_prompt_ok=true, json_mode_ok=true
- insecure-repo with --enable-llm: 9 LLM findings; qwen3.5 correctly
flagged SQLi, RCE, hardcoded credentials as critical with verbatim
evidence; 27s wall for 3 chunks
- clean-repo with --enable-llm: 0 LLM findings, 4 parsed chunks, 2.8s
- self-review with --enable-llm: 77 LLM findings + 83 static; 3 of
~30 chunks needed retry (PROMPT.md, REPORT_SCHEMA.md,
SCRUM_TEST_TEMPLATE.md — all eventually parsed); 5min wall
go vet + go test -short clean. Fixture stray.go now `package fixture`
so go-tooling doesn't choke on the orphan.
Phase D (validator cross-check) + Phase E (memory + diff/rules
subcommands) remain.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
70d68757f7
commit
e346b54e0f
+34
-14
@@ -17,17 +17,19 @@ import (
|
||||
"local-review-harness/internal/llm"
|
||||
)
|
||||
|
||||
// commonFlags wires the three flags every subcommand accepts.
|
||||
// commonFlags wires the flags every subcommand accepts.
|
||||
type commonFlags struct {
|
||||
reviewProfilePath string
|
||||
modelProfilePath string
|
||||
outputDir string
|
||||
enableLLM bool
|
||||
}
|
||||
|
||||
func bindCommonFlags(fs *flag.FlagSet, cf *commonFlags) {
|
||||
fs.StringVar(&cf.reviewProfilePath, "review-profile", "", "review profile YAML (defaults applied if empty)")
|
||||
fs.StringVar(&cf.modelProfilePath, "model-profile", "", "model profile YAML (defaults applied if empty)")
|
||||
fs.StringVar(&cf.outputDir, "output-dir", "", "override review profile output dir")
|
||||
fs.BoolVar(&cf.enableLLM, "enable-llm", false, "Phase C: also run local-Ollama LLM review (default off — static-only)")
|
||||
}
|
||||
|
||||
// resolveOutputDir picks the output dir from flag > review profile >
|
||||
@@ -64,10 +66,6 @@ func writeJSON(path string, v any) error {
|
||||
// nowUTC returns ISO-8601 UTC for receipt timestamps.
|
||||
func nowUTC() string { return time.Now().UTC().Format(time.RFC3339Nano) }
|
||||
|
||||
// Stub-only sentinel — Phase C replaces this with real Ollama provider.
|
||||
// Phase A keeps the pipeline runnable end-to-end with a degraded-status
|
||||
// model-doctor JSON.
|
||||
func nilProvider() llm.Provider { return nil }
|
||||
|
||||
// Repo runs Phase 0 (intake) + Phase 1 (static scan) + Phase 4
|
||||
// (report gen). Phase B implements the analyzers + scanner; Phase
|
||||
@@ -134,20 +132,35 @@ func ModelDoctor(args []string) int {
|
||||
outDir = rp.Reports.OutputDir
|
||||
}
|
||||
|
||||
// Phase A: stub. Phase C swaps in a real probe.
|
||||
// Phase C: real Ollama probe. Provider's HealthCheck does the
|
||||
// actual work; we package the result into the shape REPORT_SCHEMA.md
|
||||
// documents. status="ok" iff server up + at least one named
|
||||
// model loaded + basic prompt + json mode all green.
|
||||
prov := llm.NewOllama(mp.BaseURL, time.Duration(mp.TimeoutSeconds)*time.Second)
|
||||
hctx, cancel := context.WithTimeout(context.Background(), time.Duration(mp.TimeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
hs := prov.HealthCheck(hctx, mp.Model, mp.FallbackModel)
|
||||
|
||||
status := "ok"
|
||||
if !hs.ServerAvailable {
|
||||
status = "failed"
|
||||
} else if !hs.BasicPromptOK || !hs.JSONModeOK || (!hs.PrimaryModelAvailable && !hs.FallbackModelAvailable) {
|
||||
status = "degraded"
|
||||
}
|
||||
|
||||
doc := map[string]any{
|
||||
"provider": mp.Provider,
|
||||
"base_url": mp.BaseURL,
|
||||
"primary_model": mp.Model,
|
||||
"fallback_model": mp.FallbackModel,
|
||||
"server_available": false,
|
||||
"primary_model_available": false,
|
||||
"fallback_model_available": false,
|
||||
"basic_prompt_ok": false,
|
||||
"json_mode_ok": false,
|
||||
"server_available": hs.ServerAvailable,
|
||||
"primary_model_available": hs.PrimaryModelAvailable,
|
||||
"fallback_model_available": hs.FallbackModelAvailable,
|
||||
"basic_prompt_ok": hs.BasicPromptOK,
|
||||
"json_mode_ok": hs.JSONModeOK,
|
||||
"timeout_seconds": mp.TimeoutSeconds,
|
||||
"status": "degraded",
|
||||
"errors": []string{"phase A stub: real Ollama probe lands in Phase C"},
|
||||
"status": status,
|
||||
"errors": hs.Errors,
|
||||
"generated_at": nowUTC(),
|
||||
}
|
||||
out := filepath.Join(outDir, "model-doctor.json")
|
||||
@@ -156,5 +169,12 @@ func ModelDoctor(args []string) int {
|
||||
return 65
|
||||
}
|
||||
fmt.Println(out)
|
||||
return 66 // degraded exit code
|
||||
switch status {
|
||||
case "ok":
|
||||
return 0
|
||||
case "degraded":
|
||||
return 66
|
||||
default:
|
||||
return 65
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ func runRepo(ctx context.Context, repoPath string, cf commonFlags) int {
|
||||
ModelProfile: mp,
|
||||
OutputDir: outDir,
|
||||
EmitScrum: false,
|
||||
EnableLLM: cf.enableLLM,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "pipeline:", err)
|
||||
@@ -71,6 +72,7 @@ func runScrum(ctx context.Context, repoPath string, cf commonFlags) int {
|
||||
ModelProfile: mp,
|
||||
OutputDir: outDir,
|
||||
EmitScrum: true,
|
||||
EnableLLM: cf.enableLLM,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "pipeline:", err)
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
// Ollama provider — local-first per PROMPT.md.
|
||||
//
|
||||
// HealthCheck: probes /api/tags (server up + model list) + a 1-token
|
||||
// completion + a strict-JSON probe. Used by `model doctor`.
|
||||
//
|
||||
// Complete + CompleteJSON: POST /api/chat with stream=false. JSON
|
||||
// mode uses Ollama's native `format: "json"` — newer Ollama versions
|
||||
// also accept a JSON Schema there but format=json is the lowest-
|
||||
// common-denominator that works back to 0.4.
|
||||
//
|
||||
// `think: false` is set for ALL completions per the Lakehouse-Go
|
||||
// 2026-04-30 finding: qwen3.5:latest and qwen3:latest are reasoning-
|
||||
// capable but the inner-loop hot path wants direct answers, not
|
||||
// `<think>` traces consuming the token budget. Callers that NEED
|
||||
// reasoning override via opts (Phase F+, not yet wired).
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OllamaProvider is the concrete impl. Stateless; safe for concurrent
|
||||
// use (the http.Client handles connection pooling).
|
||||
type OllamaProvider struct {
|
||||
baseURL string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewOllama returns a provider pointed at baseURL. Empty baseURL
|
||||
// defaults to http://localhost:11434. timeout 0 → 120s (matches
|
||||
// model-profile default).
|
||||
func NewOllama(baseURL string, timeout time.Duration) *OllamaProvider {
|
||||
if baseURL == "" {
|
||||
baseURL = "http://localhost:11434"
|
||||
}
|
||||
if timeout == 0 {
|
||||
timeout = 120 * time.Second
|
||||
}
|
||||
return &OllamaProvider{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
httpClient: &http.Client{Timeout: timeout},
|
||||
}
|
||||
}
|
||||
|
||||
func (o *OllamaProvider) Name() string { return "ollama" }
|
||||
|
||||
// HealthCheck runs the 5 probes documented in REPORT_SCHEMA.md
|
||||
// model-doctor.json shape:
|
||||
// - server_available: GET /api/tags returns 2xx
|
||||
// - primary_model_available: name appears in tag list
|
||||
// - fallback_model_available: name appears in tag list
|
||||
// - basic_prompt_ok: a 5-token "reply OK" round-trips
|
||||
// - json_mode_ok: a JSON probe parses cleanly
|
||||
//
|
||||
// Errors surface in HealthStatus.Errors as human-readable strings
|
||||
// (no stack trace shape — operators run this from a shell).
|
||||
func (o *OllamaProvider) HealthCheck(ctx context.Context, primary, fallback string) HealthStatus {
|
||||
st := HealthStatus{Errors: []string{}}
|
||||
|
||||
// 1. Server availability + model list
|
||||
tags, err := o.listTags(ctx)
|
||||
if err != nil {
|
||||
st.Errors = append(st.Errors, "list models: "+err.Error())
|
||||
return st
|
||||
}
|
||||
st.ServerAvailable = true
|
||||
|
||||
loaded := map[string]bool{}
|
||||
for _, t := range tags {
|
||||
loaded[t] = true
|
||||
}
|
||||
st.PrimaryModelAvailable = primary != "" && loaded[primary]
|
||||
st.FallbackModelAvailable = fallback != "" && loaded[fallback]
|
||||
|
||||
// Pick the model we'll use for the live probes — primary if
|
||||
// loaded, else fallback, else the first model Ollama has.
|
||||
probeModel := ""
|
||||
switch {
|
||||
case st.PrimaryModelAvailable:
|
||||
probeModel = primary
|
||||
case st.FallbackModelAvailable:
|
||||
probeModel = fallback
|
||||
case len(tags) > 0:
|
||||
probeModel = tags[0]
|
||||
st.Errors = append(st.Errors,
|
||||
fmt.Sprintf("neither primary %q nor fallback %q loaded; using %q for liveness probe", primary, fallback, probeModel))
|
||||
default:
|
||||
st.Errors = append(st.Errors, "no models loaded; can't run liveness probe")
|
||||
return st
|
||||
}
|
||||
|
||||
// 2. Basic completion
|
||||
if got, err := o.Complete(ctx, probeModel, "Reply with the single word: OK", CompleteOptions{Temperature: 0, MaxTokens: 8, TimeoutSeconds: 30}); err != nil {
|
||||
st.Errors = append(st.Errors, "basic prompt: "+err.Error())
|
||||
} else if strings.TrimSpace(got) != "" {
|
||||
st.BasicPromptOK = true
|
||||
}
|
||||
|
||||
// 3. JSON-mode completion
|
||||
jsonGot, err := o.CompleteJSON(ctx, probeModel, `Output exactly this JSON and nothing else: {"ok": true}`, CompleteOptions{Temperature: 0, MaxTokens: 32, TimeoutSeconds: 30})
|
||||
if err != nil {
|
||||
st.Errors = append(st.Errors, "json mode: "+err.Error())
|
||||
} else {
|
||||
var probe struct{ Ok bool }
|
||||
if json.Unmarshal([]byte(jsonGot), &probe) == nil {
|
||||
st.JSONModeOK = true
|
||||
} else {
|
||||
st.Errors = append(st.Errors, "json mode: parse failed; raw="+abbrev(jsonGot, 200))
|
||||
}
|
||||
}
|
||||
|
||||
return st
|
||||
}
|
||||
|
||||
// Complete posts to /api/chat with stream=false. Returns just the
|
||||
// assistant content; token counts not surfaced (callers that need
|
||||
// them go via the chat-shape API directly, which we'll expose later).
|
||||
func (o *OllamaProvider) Complete(ctx context.Context, model, prompt string, opts CompleteOptions) (string, error) {
|
||||
body := o.chatBody(model, prompt, opts, false)
|
||||
return o.postChat(ctx, body, opts)
|
||||
}
|
||||
|
||||
// CompleteJSON requests Ollama's native JSON-mode constrained output.
|
||||
// The `format: "json"` field forces grammar-constrained generation —
|
||||
// the model can only emit valid JSON. Some models still emit garbage
|
||||
// in the content field (e.g. preamble text); validation is the
|
||||
// caller's job (PROMPT.md "AI may suggest. Code validates.").
|
||||
func (o *OllamaProvider) CompleteJSON(ctx context.Context, model, prompt string, opts CompleteOptions) (string, error) {
|
||||
body := o.chatBody(model, prompt, opts, true)
|
||||
return o.postChat(ctx, body, opts)
|
||||
}
|
||||
|
||||
func (o *OllamaProvider) chatBody(model, prompt string, opts CompleteOptions, jsonMode bool) map[string]any {
|
||||
options := map[string]any{}
|
||||
if opts.Temperature != 0 {
|
||||
options["temperature"] = opts.Temperature
|
||||
}
|
||||
if opts.MaxTokens > 0 {
|
||||
options["num_predict"] = opts.MaxTokens
|
||||
}
|
||||
body := map[string]any{
|
||||
"model": model,
|
||||
"messages": []map[string]any{
|
||||
{"role": "user", "content": prompt},
|
||||
},
|
||||
"stream": false,
|
||||
"think": false, // local hot path skips reasoning by default
|
||||
"options": options,
|
||||
}
|
||||
if jsonMode {
|
||||
body["format"] = "json"
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func (o *OllamaProvider) postChat(ctx context.Context, body map[string]any, opts CompleteOptions) (string, error) {
|
||||
bs, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", o.baseURL+"/api/chat", bytes.NewReader(bs))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
cli := o.httpClient
|
||||
if opts.TimeoutSeconds > 0 {
|
||||
cli = &http.Client{Timeout: time.Duration(opts.TimeoutSeconds) * time.Second}
|
||||
}
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return "", fmt.Errorf("ollama timeout")
|
||||
}
|
||||
return "", fmt.Errorf("ollama request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
rb, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return "", fmt.Errorf("ollama %d: %s", resp.StatusCode, abbrev(string(rb), 200))
|
||||
}
|
||||
var out struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
DoneReason string `json:"done_reason"`
|
||||
}
|
||||
if err := json.Unmarshal(rb, &out); err != nil {
|
||||
return "", fmt.Errorf("ollama decode: %w (body=%s)", err, abbrev(string(rb), 200))
|
||||
}
|
||||
return out.Message.Content, nil
|
||||
}
|
||||
|
||||
// listTags hits /api/tags and returns the loaded-model name list.
|
||||
func (o *OllamaProvider) listTags(ctx context.Context) ([]string, error) {
|
||||
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
req, _ := http.NewRequestWithContext(cctx, "GET", o.baseURL+"/api/tags", nil)
|
||||
resp, err := o.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
rb, _ := io.ReadAll(resp.Body)
|
||||
var out struct {
|
||||
Models []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"models"`
|
||||
}
|
||||
if err := json.Unmarshal(rb, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(out.Models))
|
||||
for _, m := range out.Models {
|
||||
names = append(names, m.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func abbrev(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
// Phase 2 (LLM review) implementation. Sends bounded chunks of the
|
||||
// repo to the local model, asks for strict JSON Findings, retries
|
||||
// once on parse failure, marks the phase degraded if the second
|
||||
// attempt also fails. Raw output is saved either way — operators
|
||||
// can re-parse manually if the harness rejected something useful.
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"local-review-harness/internal/analyzers"
|
||||
"local-review-harness/internal/scanner"
|
||||
)
|
||||
|
||||
// ReviewInput is one bounded review request. The harness chunks the
|
||||
// scan result into ReviewInputs (one per file or one per file-group)
|
||||
// before calling Review.
|
||||
type ReviewInput struct {
|
||||
ChunkID string // stable per-chunk identifier (file path for v0)
|
||||
Description string // human label (e.g. "internal/foo/bar.go")
|
||||
Content string // the actual code/content to review
|
||||
Language string // for the prompt context
|
||||
}
|
||||
|
||||
// ReviewOutput is what one Review call produces. RawContent is the
|
||||
// model's verbatim output before parsing — saved for forensics if
|
||||
// parsing fails.
|
||||
type ReviewOutput struct {
|
||||
ChunkID string `json:"chunk_id"`
|
||||
Findings []analyzers.Finding `json:"findings"`
|
||||
RawContent string `json:"raw_content"`
|
||||
Parsed bool `json:"parsed"`
|
||||
Retried bool `json:"retried"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Reviewer wraps a Provider with the prompt + retry logic. Stateless;
|
||||
// the prompt template is baked in for v0.
|
||||
type Reviewer struct {
|
||||
prov Provider
|
||||
model string
|
||||
opts CompleteOptions
|
||||
}
|
||||
|
||||
// NewReviewer constructs a Reviewer pointing at the configured
|
||||
// primary model. opts are passed through to every Complete call;
|
||||
// callers tune via review-profile.
|
||||
func NewReviewer(prov Provider, model string, opts CompleteOptions) *Reviewer {
|
||||
if opts.TimeoutSeconds == 0 {
|
||||
opts.TimeoutSeconds = 120
|
||||
}
|
||||
return &Reviewer{prov: prov, model: model, opts: opts}
|
||||
}
|
||||
|
||||
// Review runs the 2-attempt flow: prompt → parse → retry-with-repair-prompt → parse.
|
||||
func (r *Reviewer) Review(ctx context.Context, in ReviewInput) ReviewOutput {
|
||||
out := ReviewOutput{ChunkID: in.ChunkID}
|
||||
|
||||
// Attempt 1
|
||||
prompt := buildReviewPrompt(in, false)
|
||||
raw, err := r.prov.CompleteJSON(ctx, r.model, prompt, r.opts)
|
||||
out.RawContent = raw
|
||||
if err != nil {
|
||||
out.Error = "request failed: " + err.Error()
|
||||
return out
|
||||
}
|
||||
if findings, perr := parseFindings(raw, in); perr == nil {
|
||||
out.Findings = findings
|
||||
out.Parsed = true
|
||||
return out
|
||||
}
|
||||
|
||||
// Attempt 2 (repair prompt — feed the raw output back + ask for
|
||||
// strict JSON only). Done once; second failure is degraded.
|
||||
out.Retried = true
|
||||
repair := buildRepairPrompt(in, raw)
|
||||
raw2, err := r.prov.CompleteJSON(ctx, r.model, repair, r.opts)
|
||||
out.RawContent = raw + "\n\n---repair---\n\n" + raw2
|
||||
if err != nil {
|
||||
out.Error = "repair request failed: " + err.Error()
|
||||
return out
|
||||
}
|
||||
if findings, perr := parseFindings(raw2, in); perr == nil {
|
||||
out.Findings = findings
|
||||
out.Parsed = true
|
||||
return out
|
||||
} else {
|
||||
out.Error = "parse failed after repair: " + perr.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ReviewBatch runs Review over a slice of inputs sequentially. Could
|
||||
// parallelize at G3+, but local Ollama is GPU-bound and serial is
|
||||
// the safe v0 — burst-parallel would queue at the model server anyway.
|
||||
func (r *Reviewer) ReviewBatch(ctx context.Context, inputs []ReviewInput) []ReviewOutput {
|
||||
out := make([]ReviewOutput, 0, len(inputs))
|
||||
for _, in := range inputs {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
out = append(out, ReviewOutput{
|
||||
ChunkID: in.ChunkID,
|
||||
Error: "context cancelled before chunk processed",
|
||||
})
|
||||
continue
|
||||
default:
|
||||
}
|
||||
out = append(out, r.Review(ctx, in))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// === prompts ===
|
||||
|
||||
const reviewSystemPrompt = `You are a senior code reviewer auditing a single source file.
|
||||
|
||||
Your job: emit a JSON object with a "findings" array. Each finding
|
||||
must include:
|
||||
- title (string, < 80 chars)
|
||||
- severity ("low" | "medium" | "high" | "critical")
|
||||
- file (string, the file path you were asked to review — verbatim)
|
||||
- line_hint (string, e.g. "42" or "100-110")
|
||||
- evidence (string, a SHORT direct quote from the file — must
|
||||
exist verbatim in the source so a downstream validator can
|
||||
grep it)
|
||||
- reason (string, one sentence explaining why this is a finding)
|
||||
- suggested_fix (string, optional, one sentence)
|
||||
- confidence (number 0.0–1.0)
|
||||
|
||||
Severity guidance:
|
||||
- critical: credential leak, RCE risk, destructive command,
|
||||
unauthenticated mutation
|
||||
- high: SQL injection, broad CORS, fail-open auth, unsafe FS
|
||||
- medium: hardcoded paths, weak error handling, missing tests
|
||||
near important code
|
||||
- low: naming, duplication, doc drift
|
||||
|
||||
Hard rules (failure = your output is rejected):
|
||||
1. Output ONLY the JSON object. No prose before or after.
|
||||
2. The evidence field MUST be a verbatim substring of the file.
|
||||
If you can't quote the source, drop the finding.
|
||||
3. Don't invent file paths, line numbers, or test names.
|
||||
4. If the file is clean, return {"findings": []}.
|
||||
5. Output nothing else when you're done.`
|
||||
|
||||
func buildReviewPrompt(in ReviewInput, _ bool) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(reviewSystemPrompt)
|
||||
b.WriteString("\n\n---\n\n")
|
||||
b.WriteString("File path: ")
|
||||
b.WriteString(in.Description)
|
||||
b.WriteString("\nLanguage: ")
|
||||
b.WriteString(in.Language)
|
||||
b.WriteString("\n\nFile content:\n```\n")
|
||||
b.WriteString(in.Content)
|
||||
b.WriteString("\n```\n\nReturn JSON only.")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func buildRepairPrompt(in ReviewInput, prev string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Your previous output was not valid JSON or did not match the required schema.\n\n")
|
||||
b.WriteString("Required shape:\n")
|
||||
b.WriteString(`{"findings":[{"title":"...","severity":"...","file":"...","line_hint":"...","evidence":"...","reason":"...","confidence":0.0}]}`)
|
||||
b.WriteString("\n\nPrevious raw output (for your reference):\n")
|
||||
b.WriteString(abbrev(prev, 1500))
|
||||
b.WriteString("\n\nFor reference, the file you were reviewing was:\n")
|
||||
b.WriteString(in.Description)
|
||||
b.WriteString("\n\nReturn ONLY the JSON object now. No explanation, no markdown fences, no apology. JSON only.")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// === parsing ===
|
||||
|
||||
func parseFindings(raw string, in ReviewInput) ([]analyzers.Finding, error) {
|
||||
// Strip leading/trailing whitespace + common markdown fences.
|
||||
cleaned := strings.TrimSpace(raw)
|
||||
cleaned = strings.TrimPrefix(cleaned, "```json")
|
||||
cleaned = strings.TrimPrefix(cleaned, "```")
|
||||
cleaned = strings.TrimSuffix(cleaned, "```")
|
||||
cleaned = strings.TrimSpace(cleaned)
|
||||
if cleaned == "" {
|
||||
return nil, fmt.Errorf("empty content")
|
||||
}
|
||||
|
||||
var shell struct {
|
||||
Findings []struct {
|
||||
Title string `json:"title"`
|
||||
Severity string `json:"severity"`
|
||||
File string `json:"file"`
|
||||
LineHint string `json:"line_hint"`
|
||||
Evidence string `json:"evidence"`
|
||||
Reason string `json:"reason"`
|
||||
SuggestedFix string `json:"suggested_fix"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
} `json:"findings"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(cleaned), &shell); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal: %w", err)
|
||||
}
|
||||
|
||||
out := make([]analyzers.Finding, 0, len(shell.Findings))
|
||||
for _, f := range shell.Findings {
|
||||
sev := normalizeSeverity(f.Severity)
|
||||
if sev == "" {
|
||||
continue // model emitted a value we don't accept
|
||||
}
|
||||
// Use the chunk's file path if model omitted/lied
|
||||
filePath := f.File
|
||||
if filePath == "" {
|
||||
filePath = in.Description
|
||||
}
|
||||
out = append(out, analyzers.Finding{
|
||||
Title: truncate(f.Title, 80),
|
||||
Severity: sev,
|
||||
Status: analyzers.StatusSuspected, // validator (Phase D) promotes to confirmed
|
||||
File: filePath,
|
||||
LineHint: f.LineHint,
|
||||
Evidence: f.Evidence,
|
||||
Reason: f.Reason,
|
||||
SuggestedFix: f.SuggestedFix,
|
||||
Source: analyzers.SourceLLM,
|
||||
Confidence: clampFloat(f.Confidence, 0, 1),
|
||||
CheckID: "llm.review",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func normalizeSeverity(s string) analyzers.Severity {
|
||||
switch strings.ToLower(strings.TrimSpace(s)) {
|
||||
case "low":
|
||||
return analyzers.SeverityLow
|
||||
case "medium", "med":
|
||||
return analyzers.SeverityMedium
|
||||
case "high":
|
||||
return analyzers.SeverityHigh
|
||||
case "critical", "crit":
|
||||
return analyzers.SeverityCritical
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func clampFloat(v, lo, hi float64) float64 {
|
||||
if v < lo {
|
||||
return lo
|
||||
}
|
||||
if v > hi {
|
||||
return hi
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// === chunking ===
|
||||
|
||||
// ChunkInputsFromScan produces one ReviewInput per file under the
|
||||
// configured size limit. Files larger than maxBytes are skipped (the
|
||||
// LLM phase notes them in the receipt as "skipped: too large"). v0
|
||||
// is per-file; per-function chunking lands in Phase D+.
|
||||
func ChunkInputsFromScan(scan *scanner.Result, maxBytes int, maxChunkChars int, readFile func(abs string) string) []ReviewInput {
|
||||
out := []ReviewInput{}
|
||||
for _, f := range scan.Files {
|
||||
if f.Language == "" {
|
||||
continue // non-code files: skip LLM review (analyzers may still flag)
|
||||
}
|
||||
if f.Size > int64(maxBytes) {
|
||||
continue
|
||||
}
|
||||
content := readFile(f.Abs)
|
||||
if len(content) > maxChunkChars {
|
||||
content = content[:maxChunkChars] + "\n... (truncated for LLM context)\n"
|
||||
}
|
||||
out = append(out, ReviewInput{
|
||||
ChunkID: f.Path,
|
||||
Description: f.Path,
|
||||
Content: content,
|
||||
Language: f.Language,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Useful for callers wiring a deadline across the whole batch.
|
||||
var _ = time.Now
|
||||
@@ -9,12 +9,15 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"local-review-harness/internal/analyzers"
|
||||
"local-review-harness/internal/config"
|
||||
"local-review-harness/internal/git"
|
||||
"local-review-harness/internal/llm"
|
||||
"local-review-harness/internal/reporters"
|
||||
"local-review-harness/internal/scanner"
|
||||
)
|
||||
@@ -26,6 +29,7 @@ type Inputs struct {
|
||||
ModelProfile config.ModelProfile
|
||||
OutputDir string
|
||||
EmitScrum bool // true → also emit scrum-test/risk-register/sprint-backlog/acceptance-gates markdown
|
||||
EnableLLM bool // Phase C: actually call the model. Off by default — operators opt in.
|
||||
}
|
||||
|
||||
// Result is what the CLI shows the operator.
|
||||
@@ -100,15 +104,44 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) {
|
||||
receipt.Phases = append(receipt.Phases, staticPhase)
|
||||
res.OutputFiles = append(res.OutputFiles, "static-findings.json")
|
||||
|
||||
// --- Phase 2: LLM review (Phase C — not implemented in MVP) ---
|
||||
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
|
||||
Name: "llm_review", Status: "degraded",
|
||||
Errors: []string{"Phase C not implemented in MVP — see PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2"},
|
||||
})
|
||||
if res.ExitCode == 0 {
|
||||
// --- Phase 2: LLM review (Phase C) ---
|
||||
llmDegraded := true
|
||||
llmPhase := reporters.PhaseReceipt{Name: "llm_review", Status: "skipped"}
|
||||
if !in.EnableLLM {
|
||||
llmPhase.Errors = append(llmPhase.Errors, "LLM review not requested (pass --enable-llm to opt in)")
|
||||
} else {
|
||||
llmFindings, raw, llmErr := runLLMReview(ctx, scan, in)
|
||||
// Always save raw output, even on failure — operator forensics.
|
||||
rawPath := filepath.Join(in.OutputDir, "llm-findings.raw.json")
|
||||
if _, err := reporters.WriteJSON(rawPath, raw); err == nil {
|
||||
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.raw.json")
|
||||
}
|
||||
if llmErr != nil {
|
||||
llmPhase.Status = "degraded"
|
||||
llmPhase.Errors = append(llmPhase.Errors, llmErr.Error())
|
||||
} else {
|
||||
normalized := reporters.StaticFindings{
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Findings: llmFindings,
|
||||
Summary: reporters.SummarizeFindings(llmFindings),
|
||||
}
|
||||
if sha, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "llm-findings.normalized.json"), normalized); err == nil {
|
||||
llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.normalized.json")
|
||||
llmPhase.OutputHash = sha
|
||||
llmPhase.Status = "ok"
|
||||
llmDegraded = false
|
||||
findings = append(findings, llmFindings...)
|
||||
res.OutputFiles = append(res.OutputFiles, "llm-findings.raw.json", "llm-findings.normalized.json")
|
||||
} else {
|
||||
llmPhase.Status = "failed"
|
||||
llmPhase.Errors = append(llmPhase.Errors, "write normalized: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
if llmDegraded && res.ExitCode == 0 {
|
||||
res.ExitCode = 66
|
||||
}
|
||||
llmDegraded := true
|
||||
receipt.Phases = append(receipt.Phases, llmPhase)
|
||||
|
||||
// --- Phase 3: validation (Phase D — also deferred) ---
|
||||
receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{
|
||||
@@ -175,6 +208,50 @@ func writeReceipt(outputDir string, r *reporters.Receipt, startedAt time.Time, _
|
||||
return err
|
||||
}
|
||||
|
||||
// runLLMReview chunks the scan into per-file inputs, calls the
|
||||
// reviewer, and aggregates parsed findings + raw outputs. Returns
|
||||
// (findings, raw-outputs-array-for-receipts, error). The error is
|
||||
// non-nil only when the provider is fundamentally unreachable;
|
||||
// per-chunk parse failures land as ReviewOutput.Error and don't
|
||||
// fail the whole phase.
|
||||
func runLLMReview(ctx context.Context, scan *scanner.Result, in Inputs) ([]analyzers.Finding, []llm.ReviewOutput, error) {
|
||||
prov := llm.NewOllama(in.ModelProfile.BaseURL, time.Duration(in.ModelProfile.TimeoutSeconds)*time.Second)
|
||||
hctx, hcancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer hcancel()
|
||||
hs := prov.HealthCheck(hctx, in.ModelProfile.Model, in.ModelProfile.FallbackModel)
|
||||
if !hs.ServerAvailable {
|
||||
return nil, nil, fmt.Errorf("ollama unreachable at %s — Phase 2 cannot run", in.ModelProfile.BaseURL)
|
||||
}
|
||||
if !hs.PrimaryModelAvailable && !hs.FallbackModelAvailable {
|
||||
return nil, nil, fmt.Errorf("neither primary %q nor fallback %q loaded in Ollama", in.ModelProfile.Model, in.ModelProfile.FallbackModel)
|
||||
}
|
||||
model := in.ModelProfile.Model
|
||||
if !hs.PrimaryModelAvailable {
|
||||
model = in.ModelProfile.FallbackModel
|
||||
}
|
||||
|
||||
r := llm.NewReviewer(prov, model, llm.CompleteOptions{
|
||||
Temperature: in.ModelProfile.Temperature,
|
||||
MaxTokens: 0, // let model decide
|
||||
TimeoutSeconds: in.ModelProfile.TimeoutSeconds,
|
||||
})
|
||||
|
||||
chunks := llm.ChunkInputsFromScan(scan, in.ReviewProfile.Limits.MaxFileBytes, in.ReviewProfile.Limits.MaxLLMChunkChars, func(abs string) string {
|
||||
b, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
})
|
||||
|
||||
outputs := r.ReviewBatch(ctx, chunks)
|
||||
findings := []analyzers.Finding{}
|
||||
for _, o := range outputs {
|
||||
findings = append(findings, o.Findings...)
|
||||
}
|
||||
return findings, outputs, nil
|
||||
}
|
||||
|
||||
func newRunID(t time.Time) string {
|
||||
var rb [4]byte
|
||||
_, _ = rand.Read(rb[:])
|
||||
|
||||
Reference in New Issue
Block a user