matrix: judge-gated Shape B inject — closes lift-suite tail issues
Lift suite run #004 left two unresolved tail issues: - Q6 ("Forklift loader") ↔ Q7 ("Hazmat warehouse, cold storage") swap recordings as warm top-1 because their embeddings are within 0.20 cosine of each other. Distance gate can't tell them apart. - Q9 + Q15 lose paraphrase recovery when qwen2.5 rephrases past the 0.20 threshold. Distance says "drift too far"; sometimes the drift is real (skip), sometimes the paraphrase is still on-domain (don't want to skip). Multi-coord run #008's judge re-rating proved the LLM can distinguish: Q3 crane case landed at distance 0.23 (looks tight) but rating 1 (irrelevant). The judge sees domain mismatch the embedder doesn't. This commit lifts that pattern into the matrix substrate. Shape B inject now optionally routes every candidate through a judge gate before the rank insert lands. Distance + judge BOTH have to approve. internal/matrix/playbook.go: - InjectPlaybookMisses signature gains a query string + an optional InjectGate. nil gate preserves pre-judge-gating behavior (current tests already pass with nil). - New InjectGate interface + InjectGateFunc adapter for tests and non-LLM callers. - Per-candidate gate.Approve(query, hit) call inserted between the dedup and the inject. Rejected candidates skip silently; injected count reflects post-gate decision. internal/matrix/judge.go (new, ~140 lines): - LLMJudgeGate calls an Ollama-shape /api/chat endpoint with the same 1-5 staffing-rubric prompt that worked in multi_coord run #008. fail-closed on HTTP/JSON errors (don't inject if judge can't speak — better miss than wrong-domain). - NewLLMJudgeGate returns nil when URL or Model is empty, matching InjectGate's nil-means-no-judge semantics. internal/matrix/retrieve.go: - SearchRequest gains JudgeURL, JudgeModel, JudgeMinRating fields. Run() builds an LLMJudgeGate when set; passes nil otherwise. Backward compatible — existing callers see no behavior change. Tests: - TestInjectPlaybookMisses_GateRejectsCandidate (rejectAll → 0 injected, even with tight distance) - TestInjectPlaybookMisses_GateApprovesCandidate (approveAll → same as nil-gate behavior) - TestInjectPlaybookMisses_GateSeesCorrectQuery (gate receives CURRENT query + RECORDED query separately so it can score the (current, candidate) pair) - All 5 existing inject tests updated to new signature go test ./internal/matrix → all 8 inject tests pass. go test ./internal/matrix ./internal/shared ./cmd/{matrixd, queryd,pathwayd,observerd} → all green. STATE_OF_PLAY: - OPEN item #1 (judge-gated injection) closed. - DO NOT RELITIGATE adds the substrate-level judge-gate lock. - OPEN list now 5 rows (was 6). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -187,7 +187,7 @@ func TestInjectPlaybookMisses_AddsMissingAnswers(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0)
|
||||
out, injected := InjectPlaybookMisses("test query", results, hits, 0, nil)
|
||||
if injected != 1 {
|
||||
t.Fatalf("expected 1 injected, got %d", injected)
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func TestInjectPlaybookMisses_SkipsAnswersAlreadyPresent(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0)
|
||||
out, injected := InjectPlaybookMisses("test query", results, hits, 0, nil)
|
||||
if injected != 0 {
|
||||
t.Errorf("expected 0 injected (answer already present), got %d", injected)
|
||||
}
|
||||
@@ -266,7 +266,7 @@ func TestInjectPlaybookMisses_DedupesPerAnswer(t *testing.T) {
|
||||
Entry: PlaybookEntry{QueryText: "q2", AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0},
|
||||
},
|
||||
}
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0.5) // explicit loose threshold so 0.30 hits qualify
|
||||
out, injected := InjectPlaybookMisses("test query", results, hits, 0.5, nil) // explicit loose threshold so 0.30 hits qualify
|
||||
if injected != 1 {
|
||||
t.Errorf("expected 1 injection (deduped), got %d", injected)
|
||||
}
|
||||
@@ -280,6 +280,89 @@ func TestInjectPlaybookMisses_DedupesPerAnswer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectPlaybookMisses_GateRejectsCandidate locks the judge-gate
|
||||
// path (OPEN item #1, closed by this commit). When the InjectGate
|
||||
// returns false on a candidate, the candidate is skipped — even if
|
||||
// distance would otherwise allow it. Closes the lift-suite tail
|
||||
// issues where embedding said "tight" but a judge said "wrong domain."
|
||||
func TestInjectPlaybookMisses_GateRejectsCandidate(t *testing.T) {
|
||||
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
|
||||
hits := []PlaybookHit{
|
||||
{
|
||||
PlaybookID: "pb-x",
|
||||
Distance: 0.10, // tight in cosine — would inject without gate
|
||||
Entry: PlaybookEntry{
|
||||
QueryText: "recorded crane operator query",
|
||||
AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0,
|
||||
},
|
||||
},
|
||||
}
|
||||
rejectAll := InjectGateFunc(func(string, PlaybookHit) bool { return false })
|
||||
out, injected := InjectPlaybookMisses("forklift loader query", results, hits, 0, rejectAll)
|
||||
if injected != 0 {
|
||||
t.Errorf("rejectAll gate should skip injection, got %d injected", injected)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Errorf("results should be unchanged at len=1, got %d", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectPlaybookMisses_GateApprovesCandidate locks the
|
||||
// always-approve gate path: behavior matches nil-gate (current
|
||||
// distance-only filter). Useful for tests that want to assert
|
||||
// "judge-gate API is wired" without an actual decision.
|
||||
func TestInjectPlaybookMisses_GateApprovesCandidate(t *testing.T) {
|
||||
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
|
||||
hits := []PlaybookHit{
|
||||
{
|
||||
PlaybookID: "pb-x",
|
||||
Distance: 0.10,
|
||||
Entry: PlaybookEntry{
|
||||
QueryText: "x", AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0,
|
||||
},
|
||||
},
|
||||
}
|
||||
approveAll := InjectGateFunc(func(string, PlaybookHit) bool { return true })
|
||||
out, injected := InjectPlaybookMisses("test query", results, hits, 0, approveAll)
|
||||
if injected != 1 {
|
||||
t.Errorf("approveAll gate should inject, got %d", injected)
|
||||
}
|
||||
if len(out) != 2 {
|
||||
t.Errorf("results should grow to 2, got %d", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectPlaybookMisses_GateSeesCorrectQuery locks the gate's
|
||||
// query+hit visibility — the gate must receive the CURRENT search's
|
||||
// query (not the recorded one) so it can judge the (current_query,
|
||||
// candidate) pair. The recorded query lives on hit.Entry.QueryText.
|
||||
func TestInjectPlaybookMisses_GateSeesCorrectQuery(t *testing.T) {
|
||||
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
|
||||
hits := []PlaybookHit{
|
||||
{
|
||||
PlaybookID: "pb-x",
|
||||
Distance: 0.10,
|
||||
Entry: PlaybookEntry{
|
||||
QueryText: "RECORDED",
|
||||
AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0,
|
||||
},
|
||||
},
|
||||
}
|
||||
var seenQuery, seenRecordedQuery string
|
||||
gate := InjectGateFunc(func(q string, h PlaybookHit) bool {
|
||||
seenQuery = q
|
||||
seenRecordedQuery = h.Entry.QueryText
|
||||
return true
|
||||
})
|
||||
_, _ = InjectPlaybookMisses("CURRENT", results, hits, 0, gate)
|
||||
if seenQuery != "CURRENT" {
|
||||
t.Errorf("gate received query=%q, want CURRENT", seenQuery)
|
||||
}
|
||||
if seenRecordedQuery != "RECORDED" {
|
||||
t.Errorf("gate received recorded=%q, want RECORDED", seenRecordedQuery)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectPlaybookMisses_RespectsInjectThreshold locks the
|
||||
// cross-pollination defense added after run #003: hits whose playbook
|
||||
// distance exceeds the inject threshold are skipped, preventing the
|
||||
@@ -303,7 +386,7 @@ func TestInjectPlaybookMisses_RespectsInjectThreshold(t *testing.T) {
|
||||
},
|
||||
}
|
||||
// Default threshold (0 → DefaultPlaybookMaxInjectDistance = 0.20)
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0)
|
||||
out, injected := InjectPlaybookMisses("test query", results, hits, 0, nil)
|
||||
if injected != 1 {
|
||||
t.Errorf("expected 1 injection (only the tight hit qualifies), got %d", injected)
|
||||
}
|
||||
@@ -324,7 +407,7 @@ func TestInjectPlaybookMisses_RespectsInjectThreshold(t *testing.T) {
|
||||
// TestInjectPlaybookMisses_EmptyHits is a fast-path no-op check.
|
||||
func TestInjectPlaybookMisses_EmptyHits(t *testing.T) {
|
||||
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
|
||||
out, injected := InjectPlaybookMisses(results, nil, 0)
|
||||
out, injected := InjectPlaybookMisses("test query", results, nil, 0, nil)
|
||||
if injected != 0 {
|
||||
t.Errorf("expected 0 injection, got %d", injected)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user