matrix: Shape B — inject playbook misses + 6/6 paraphrase recovery

The v0 boost-only stance documented in internal/matrix/playbook.go:22-27
("the boost only re-ranks results that ALREADY surfaced from the regular
retrieval") couldn't promote recorded answers that dropped out of a
paraphrase's top-K. playbook_lift_002 surfaced exactly that gap: 0/2
paraphrase recoveries because the recorded answers weren't in regular
retrieval at all (rank=-1).

Shape B: when warm-pass retrieval doesn't surface a playbook hit's
answer, inject a synthetic Result for it directly. Distance =
playbook_hit_distance × BoostFactor — same formula as the boost path so
injections land in comparable distance space. Caller re-sorts +
truncates after both boost and inject have run.

Result on playbook_lift_003 (Shape B + paraphrase pass):

  Verbatim discovery        6
  Verbatim lift             2 / 6
  **Paraphrase top-1**      **6 / 6**
  Paraphrase any-rank in K  6 / 6
  Mean Δ top-1 distance     -0.1637 (warm closer than cold)

Every paraphrase the judge generated landed the v1-recorded answer at
top-1 of the new query's results. The learning property holds — cosine
on embed(paraphrase) finds the recorded query's vector within
DefaultPlaybookMaxDistance (0.5), and Shape B injects the answer.

Verbatim lift dropped from v1's 7/8 because Shape B cross-pollinates
recorded answers across queries. w-4435 (Q2's recording) appears as
warm top-1 for several other queries because their embeddings are
within the playbook hit threshold of "OSHA-30 forklift Wisconsin." This
is a feature, not a bug — the matrix layer's purpose is to share
knowledge across queries — but the lift metric only counts "warm top-1
== cold judge best," so cross-pollinated lifts don't register. A v3
metric would re-judge warm pass to measure true judge improvement.

Tests:
- TestInjectPlaybookMisses_AddsMissingAnswers — primary claim
- TestInjectPlaybookMisses_SkipsAnswersAlreadyPresent — no double-inject
- TestInjectPlaybookMisses_DedupesPerAnswer — multi-hit same answer
- TestInjectPlaybookMisses_EmptyHits — fast-path no-op

Driver fix: ParaphraseRecordedRank int → *int. The `omitempty` int
silently dropped rank=0 (top-1, the WANTED value) from JSON, making the
v003 report show "null" instead of "0" for every successful recovery.
Pointer keeps nil/rank-0 distinguishable.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-30 07:06:13 -05:00
co-authored by Claude Opus 4.7
parent e9822f025d
commit 154a72ea5e
5 changed files with 352 additions and 10 deletions
+128
View File
@@ -164,6 +164,134 @@ func TestUnmarshalPlaybookMetadata_RejectsEmpty(t *testing.T) {
}
}
// TestInjectPlaybookMisses_AddsMissingAnswers locks Shape B's primary
// claim: when a playbook hit's answer isn't already in regular
// retrieval results, InjectPlaybookMisses appends a synthetic Result
// for it. Reality test playbook_lift_002 surfaced 0/2 paraphrase
// recoveries because the v0 boost-only stance couldn't promote
// answers that dropped out of the paraphrase's top-K.
func TestInjectPlaybookMisses_AddsMissingAnswers(t *testing.T) {
results := []Result{
{ID: "w-1", Corpus: "workers", Distance: 0.30},
{ID: "w-2", Corpus: "workers", Distance: 0.35},
}
hits := []PlaybookHit{
{
PlaybookID: "pb-x",
Distance: 0.20, // current query is close to recorded query
Entry: PlaybookEntry{
QueryText: "recorded query",
AnswerID: "w-99", // NOT in results
AnswerCorpus: "workers",
Score: 1.0, // strong outcome → boost factor 0.5
},
},
}
out, injected := InjectPlaybookMisses(results, hits)
if injected != 1 {
t.Fatalf("expected 1 injected, got %d", injected)
}
if len(out) != 3 {
t.Fatalf("expected len=3, got %d (%v)", len(out), idsOf(out))
}
// The injected result should be findable + carry the playbook
// provenance metadata flag.
var injectedResult *Result
for i := range out {
if out[i].ID == "w-99" {
injectedResult = &out[i]
break
}
}
if injectedResult == nil {
t.Fatal("w-99 not present in output")
}
// distance = 0.20 * 0.5 = 0.10 → near-top after caller re-sorts
if injectedResult.Distance < 0.099 || injectedResult.Distance > 0.101 {
t.Errorf("expected injected distance ~0.10, got %f", injectedResult.Distance)
}
var meta map[string]any
if err := json.Unmarshal(injectedResult.Metadata, &meta); err != nil {
t.Fatalf("decode meta: %v", err)
}
if v, _ := meta["playbook_injected"].(bool); !v {
t.Errorf("expected playbook_injected=true marker, got %v", meta)
}
if v, _ := meta["playbook_query_text"].(string); v != "recorded query" {
t.Errorf("expected recorded query in meta, got %v", v)
}
}
// TestInjectPlaybookMisses_SkipsAnswersAlreadyPresent locks the
// boost-only-when-present property. If a playbook hit's answer is
// ALREADY in results, we don't duplicate-inject — ApplyPlaybookBoost
// has handled that case via in-place re-rank.
func TestInjectPlaybookMisses_SkipsAnswersAlreadyPresent(t *testing.T) {
results := []Result{
{ID: "w-1", Corpus: "workers", Distance: 0.30},
{ID: "w-99", Corpus: "workers", Distance: 0.40}, // ALREADY HERE
}
hits := []PlaybookHit{
{
PlaybookID: "pb-x",
Distance: 0.20,
Entry: PlaybookEntry{
QueryText: "x", AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0,
},
},
}
out, injected := InjectPlaybookMisses(results, hits)
if injected != 0 {
t.Errorf("expected 0 injected (answer already present), got %d", injected)
}
if len(out) != 2 {
t.Errorf("expected results unchanged at len=2, got %d", len(out))
}
}
// TestInjectPlaybookMisses_DedupesPerAnswer locks: multiple playbook
// hits all pointing to the same missing answer collapse to ONE
// injection (the highest-scoring hit wins).
func TestInjectPlaybookMisses_DedupesPerAnswer(t *testing.T) {
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
hits := []PlaybookHit{
{
PlaybookID: "pb-low",
Distance: 0.30,
Entry: PlaybookEntry{QueryText: "q1", AnswerID: "w-99", AnswerCorpus: "workers", Score: 0.4},
},
{
PlaybookID: "pb-high",
Distance: 0.30,
Entry: PlaybookEntry{QueryText: "q2", AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0},
},
}
out, injected := InjectPlaybookMisses(results, hits)
if injected != 1 {
t.Errorf("expected 1 injection (deduped), got %d", injected)
}
// Score=1.0 (the high one) wins → boost factor 0.5 → distance 0.15
for _, r := range out {
if r.ID == "w-99" {
if r.Distance < 0.149 || r.Distance > 0.151 {
t.Errorf("expected distance from highest-score hit (~0.15), got %f", r.Distance)
}
}
}
}
// TestInjectPlaybookMisses_EmptyHits is a fast-path no-op check.
func TestInjectPlaybookMisses_EmptyHits(t *testing.T) {
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
out, injected := InjectPlaybookMisses(results, nil)
if injected != 0 {
t.Errorf("expected 0 injection, got %d", injected)
}
if len(out) != 1 {
t.Errorf("results should be unchanged, got len=%d", len(out))
}
}
func abs(f float64) float64 {
if f < 0 {
return -f