matrix: split boost / inject thresholds — kills Shape B cross-pollination
Run #003 surfaced rampant cross-pollination: Q2's "OSHA-30 forklift Wisconsin" recording (w-4435) became warm top-1 for Q19 (dental hygienist), Q20 (RN), Q21 (software engineer), and 6 other unrelated staffing queries. Cause: InjectPlaybookMisses inherited the same DefaultPlaybookMaxDistance (0.5) as the boost path, but inject is structurally riskier than boost — boost only re-ranks results that already retrieved on their own merits, while inject FORCES a result into top-K, so a loose match cross-pollinates wrong-domain answers. Empirical motivation from v3: Implied playbook hit distances for cross-pollinated cases: 0.20-0.46 Implied distances for the 6/6 paraphrase recoveries: 0.23-0.30 Threshold of 0.20 should keep most paraphrases, kill the OOD bleed. Implementation: - New DefaultPlaybookMaxInjectDistance = 0.20 in playbook.go. - New PlaybookMaxInjectDistance field on SearchRequest (override). - InjectPlaybookMisses signature gains maxInjectDist param; hits whose Distance exceeds it are skipped (boost path may still re-rank them). - TestInjectPlaybookMisses_RespectsInjectThreshold locks the contract with one tight + one loose hit, asserting only the tight one injects. - Existing tests pass explicit threshold (0 = default for tight tests, 0.5 for the dedupe test which uses 0.30 hits). Run #004 result on identical queries with the split threshold: Verbatim discovery 8 (vs v3's 6 — judge variance, separate) Verbatim lift 6 / 8 (75%) Paraphrase top-1 6 / 8 (75%) Paraphrase any-rank in K 6 / 8 OOD queries Q19/Q20/Q21 ALL show warm top-1 = cold top-1 (no injection) — cross-pollination eliminated where it was wrong-direction. Mean Δ top-1 distance dropped from -0.164 (v3, distorted) to -0.071 (v4, comparable to v1's -0.053). Two paraphrases missed in v4 (Q9, Q15) were ones where qwen2.5 rephrased liberally enough to drift past 0.20 — Q9: "Inventory specialist..." → "Individual needed for inventory management..." and Q15: "Engaged warehouse associate..." → "Warehouse associate currently engaged with a robust history...". The system correctly refusing to inject when it's not confident is the right product behavior; the boost path still re-ranks recorded answers when they appear in regular retrieval. The Q6 ↔ Q7 cross-pollination ("Forklift-certified loader" ↔ "Hazmat warehouse worker") is legitimate — these are genuinely similar staffing queries and the judge ranks both directions as plausible. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -187,7 +187,7 @@ func TestInjectPlaybookMisses_AddsMissingAnswers(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
out, injected := InjectPlaybookMisses(results, hits)
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0)
|
||||
if injected != 1 {
|
||||
t.Fatalf("expected 1 injected, got %d", injected)
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func TestInjectPlaybookMisses_SkipsAnswersAlreadyPresent(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
out, injected := InjectPlaybookMisses(results, hits)
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0)
|
||||
if injected != 0 {
|
||||
t.Errorf("expected 0 injected (answer already present), got %d", injected)
|
||||
}
|
||||
@@ -266,7 +266,7 @@ func TestInjectPlaybookMisses_DedupesPerAnswer(t *testing.T) {
|
||||
Entry: PlaybookEntry{QueryText: "q2", AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0},
|
||||
},
|
||||
}
|
||||
out, injected := InjectPlaybookMisses(results, hits)
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0.5) // explicit loose threshold so 0.30 hits qualify
|
||||
if injected != 1 {
|
||||
t.Errorf("expected 1 injection (deduped), got %d", injected)
|
||||
}
|
||||
@@ -280,10 +280,51 @@ func TestInjectPlaybookMisses_DedupesPerAnswer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectPlaybookMisses_RespectsInjectThreshold locks the
|
||||
// cross-pollination defense added after run #003: hits whose playbook
|
||||
// distance exceeds the inject threshold are skipped, preventing the
|
||||
// "OSHA-30 forklift" recording from surfacing as warm top-1 for an
|
||||
// unrelated dental-hygienist query just because their text vectors
|
||||
// happened to fall within boost-threshold (0.5).
|
||||
func TestInjectPlaybookMisses_RespectsInjectThreshold(t *testing.T) {
|
||||
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
|
||||
// Two hits: one within tight inject threshold, one beyond it but
|
||||
// within boost threshold. Only the tight one should inject.
|
||||
hits := []PlaybookHit{
|
||||
{
|
||||
PlaybookID: "tight",
|
||||
Distance: 0.10, // within inject (true paraphrase territory)
|
||||
Entry: PlaybookEntry{QueryText: "q1", AnswerID: "w-tight", AnswerCorpus: "workers", Score: 1.0},
|
||||
},
|
||||
{
|
||||
PlaybookID: "loose",
|
||||
Distance: 0.40, // boost-eligible but inject-rejected
|
||||
Entry: PlaybookEntry{QueryText: "q2", AnswerID: "w-loose", AnswerCorpus: "workers", Score: 1.0},
|
||||
},
|
||||
}
|
||||
// Default threshold (0 → DefaultPlaybookMaxInjectDistance = 0.20)
|
||||
out, injected := InjectPlaybookMisses(results, hits, 0)
|
||||
if injected != 1 {
|
||||
t.Errorf("expected 1 injection (only the tight hit qualifies), got %d", injected)
|
||||
}
|
||||
gotTight := false
|
||||
for _, r := range out {
|
||||
if r.ID == "w-tight" {
|
||||
gotTight = true
|
||||
}
|
||||
if r.ID == "w-loose" {
|
||||
t.Errorf("loose hit (distance > inject threshold) was injected anyway")
|
||||
}
|
||||
}
|
||||
if !gotTight {
|
||||
t.Error("tight hit should have been injected")
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectPlaybookMisses_EmptyHits is a fast-path no-op check.
|
||||
func TestInjectPlaybookMisses_EmptyHits(t *testing.T) {
|
||||
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
|
||||
out, injected := InjectPlaybookMisses(results, nil)
|
||||
out, injected := InjectPlaybookMisses(results, nil, 0)
|
||||
if injected != 0 {
|
||||
t.Errorf("expected 0 injection, got %d", injected)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user