profit/golangLAKEHOUSE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

matrix: cross-role playbook gate — closes real_001 bleed (OPEN #1)

Browse Source 
real_001 surfaced same-client+city queries bleeding across roles:
Q#2 (Forklift Operator @ Beacon Freight Detroit) recorded e-6193
in the playbook corpus. Q#5 (Pickers same client+city) and Q#10
(CNC Operator same client+city) embedded within 0.13-0.18 cosine of
Q#2's query — well inside the 0.20 inject threshold — so e-6193
injected on both, demoting the cold-pass-correct workers.

Root cause: the inject distance threshold isn't tight enough on
the same-client+city cluster. Cosine collapses queries that share
city + client + count-token + time-token regardless of role. The
existing judge gate is per-injection at record time and doesn't
fire at retrieve time.

Fix: structural role gate in front of both Shape A boost and
Shape B inject. PlaybookEntry gains Role; SearchRequest gains
QueryRole. When both are non-empty and differ under roleEqual's
case+plural normalization, the entry is rejected before BoostFactor
or judge-gate logic runs.

Backward-compat: empty role on either side disables the gate —
preserves behavior for the lift suite's free-form multi-constraint
queries that have no clean single role. Caller-supplied (not
inferred), so existing recordings unaffected.

Wire-through:
- internal/matrix/playbook.go: Role field, NewPlaybookEntryWithRole,
  roleEqual helper with plural+case normalization
- internal/matrix/retrieve.go: QueryRole on SearchRequest, threaded
  to both ApplyPlaybookBoost + InjectPlaybookMisses
- cmd/matrixd/main.go: role on POST /matrix/playbooks/record + bulk
- scripts/playbook_lift/main.go: extractRoleFromNeed regex pulls
  role from "Need N {role}{s} in" queries (the fill_events shape);
  free-form queries fall back to empty (gate disabled)

Tests (5 new):
- TestInjectPlaybookMisses_RoleGateRejectsCrossRole: exact Q#10
  scenario (distance 0.135, recorded "Forklift Operator", query
  "CNC Operator") — locks the bleed at unit level
- TestInjectPlaybookMisses_RoleGateAllowsSameRole: Forklift Operator
  recording fires on Forklift Operators query (plural normalization)
- TestInjectPlaybookMisses_RoleGateBackwardCompat: empty Role on
  either side = gate disabled, preserves current behavior
- TestApplyPlaybookBoost_RoleGateRejectsCrossRole: Shape A defense
  in depth — boost doesn't fire on cross-role even when answer is
  in cold top-K
- TestRoleEqual_PluralAndCase: case + -s + -es plural normalization

Verification (real_002, same query set as real_001):
- Q#5 Pickers @ Beacon Freight: e-6193 → e-8499 (no bleed)
- Q#10 CNC Operator @ Beacon Freight: e-6193 → w-2404 (no bleed)
- Discoveries + lifts unchanged at 2 each (same-role lift still fires)
- Mean Δdist tightens from -0.127 to -0.040 (boosts no longer
  pulling distances through the floor on cross-role mismatches)

Findings: reports/reality-tests/real_002_findings.md.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-30 20:34:10 -05:00
co-authored by Claude Opus 4.7
parent 7f2f112e6a
commit 997527be4d
8 changed files with 513 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/matrix/playbook_test.go
+152 -12
View File
@@ -54,7 +54,7 @@ func TestApplyPlaybookBoost_NoHitsLeaveResultsAlone(t *testing.T) {
{ID: "a", Distance: 0.1, Corpus: "x"},
{ID: "b", Distance: 0.2, Corpus: "x"},
}
n := ApplyPlaybookBoost(results, nil)
n := ApplyPlaybookBoost("", results, nil)
if n != 0 {
t.Errorf("expected 0 boosted, got %d", n)
}
@@ -77,7 +77,7 @@ func TestApplyPlaybookBoost_BoostMovesResultUp(t *testing.T) {
AnswerID: "c", AnswerCorpus: "x", Score: 1.0,
}},
}
n := ApplyPlaybookBoost(results, hits)
n := ApplyPlaybookBoost("", results, hits)
if n != 1 {
t.Errorf("expected 1 boosted, got %d", n)
}
@@ -104,7 +104,7 @@ func TestApplyPlaybookBoost_HighestScoreWinsForSameAnswer(t *testing.T) {
AnswerID: "a", AnswerCorpus: "x", Score: 0.9,
}},
}
ApplyPlaybookBoost(results, hits)
ApplyPlaybookBoost("", results, hits)
wantDist := 0.30 * 0.55
if abs(float64(results[0].Distance)-wantDist) > 1e-6 {
t.Errorf("strong-score boost should win: want %.4f, got %.4f", wantDist, results[0].Distance)
@@ -125,7 +125,7 @@ func TestApplyPlaybookBoost_CorpusAttributionRespected(t *testing.T) {
AnswerID: "a", AnswerCorpus: "x", Score: 1.0,
}},
}
n := ApplyPlaybookBoost(results, hits)
n := ApplyPlaybookBoost("", results, hits)
if n != 0 {
t.Errorf("cross-corpus collision should not boost: got %d", n)
}
@@ -187,7 +187,7 @@ func TestInjectPlaybookMisses_AddsMissingAnswers(t *testing.T) {
},
},
}
out, injected := InjectPlaybookMisses("test query", results, hits, 0, nil)
out, injected := InjectPlaybookMisses("test query", "", results, hits, 0, nil)
if injected != 1 {
t.Fatalf("expected 1 injected, got %d", injected)
}
@@ -240,7 +240,7 @@ func TestInjectPlaybookMisses_SkipsAnswersAlreadyPresent(t *testing.T) {
},
},
}
out, injected := InjectPlaybookMisses("test query", results, hits, 0, nil)
out, injected := InjectPlaybookMisses("test query", "", results, hits, 0, nil)
if injected != 0 {
t.Errorf("expected 0 injected (answer already present), got %d", injected)
}
@@ -266,7 +266,7 @@ func TestInjectPlaybookMisses_DedupesPerAnswer(t *testing.T) {
Entry: PlaybookEntry{QueryText: "q2", AnswerID: "w-99", AnswerCorpus: "workers", Score: 1.0},
},
}
out, injected := InjectPlaybookMisses("test query", results, hits, 0.5, nil) // explicit loose threshold so 0.30 hits qualify
out, injected := InjectPlaybookMisses("test query", "", results, hits, 0.5, nil) // explicit loose threshold so 0.30 hits qualify
if injected != 1 {
t.Errorf("expected 1 injection (deduped), got %d", injected)
}
@@ -298,7 +298,7 @@ func TestInjectPlaybookMisses_GateRejectsCandidate(t *testing.T) {
},
}
rejectAll := InjectGateFunc(func(string, PlaybookHit) bool { return false })
out, injected := InjectPlaybookMisses("forklift loader query", results, hits, 0, rejectAll)
out, injected := InjectPlaybookMisses("forklift loader query", "", results, hits, 0, rejectAll)
if injected != 0 {
t.Errorf("rejectAll gate should skip injection, got %d injected", injected)
}
@@ -323,7 +323,7 @@ func TestInjectPlaybookMisses_GateApprovesCandidate(t *testing.T) {
},
}
approveAll := InjectGateFunc(func(string, PlaybookHit) bool { return true })
out, injected := InjectPlaybookMisses("test query", results, hits, 0, approveAll)
out, injected := InjectPlaybookMisses("test query", "", results, hits, 0, approveAll)
if injected != 1 {
t.Errorf("approveAll gate should inject, got %d", injected)
}
@@ -354,7 +354,7 @@ func TestInjectPlaybookMisses_GateSeesCorrectQuery(t *testing.T) {
seenRecordedQuery = h.Entry.QueryText
return true
})
_, _ = InjectPlaybookMisses("CURRENT", results, hits, 0, gate)
_, _ = InjectPlaybookMisses("CURRENT", "", results, hits, 0, gate)
if seenQuery != "CURRENT" {
t.Errorf("gate received query=%q, want CURRENT", seenQuery)
}
@@ -386,7 +386,7 @@ func TestInjectPlaybookMisses_RespectsInjectThreshold(t *testing.T) {
},
}
// Default threshold (0 → DefaultPlaybookMaxInjectDistance = 0.20)
out, injected := InjectPlaybookMisses("test query", results, hits, 0, nil)
out, injected := InjectPlaybookMisses("test query", "", results, hits, 0, nil)
if injected != 1 {
t.Errorf("expected 1 injection (only the tight hit qualifies), got %d", injected)
}
@@ -407,7 +407,7 @@ func TestInjectPlaybookMisses_RespectsInjectThreshold(t *testing.T) {
// TestInjectPlaybookMisses_EmptyHits is a fast-path no-op check.
func TestInjectPlaybookMisses_EmptyHits(t *testing.T) {
results := []Result{{ID: "w-1", Corpus: "workers", Distance: 0.30}}
out, injected := InjectPlaybookMisses("test query", results, nil, 0, nil)
out, injected := InjectPlaybookMisses("test query", "", results, nil, 0, nil)
if injected != 0 {
t.Errorf("expected 0 injection, got %d", injected)
}
@@ -416,6 +416,146 @@ func TestInjectPlaybookMisses_EmptyHits(t *testing.T) {
}
}
// Cross-role gate tests — reproduce the real_001 bleed scenario at
// the unit-test level so regressions can't ship without notice.
//
// real_001 scenario: Q#2 records playbook entry for e-6193 fitting
// "Forklift Operator at Beacon Freight Detroit MI". Q#10 (CNC Operator
// at same client + city) embeds within 0.135 cosine of Q#2's
// playbook entry — well inside the 0.20 inject threshold. Without the
// role gate, e-6193 injects on Q#10 at warm-top-1, demoting the
// cold-pass-correct w-3759.
func TestInjectPlaybookMisses_RoleGateRejectsCrossRole(t *testing.T) {
results := []Result{{ID: "w-3759", Corpus: "workers", Distance: 0.50}}
hits := []PlaybookHit{{
PlaybookID: "pb-q2",
Distance: 0.135, // tight; well within DefaultPlaybookMaxInjectDistance
Entry: PlaybookEntry{
QueryText: "Need 1 Forklift Operator in Detroit MI ...",
Role: "Forklift Operator",
AnswerID: "e-6193",
AnswerCorpus: "ethereal_workers",
Score: 1.0,
},
}}
out, injected := InjectPlaybookMisses(
"Need 1 CNC Operator in Detroit MI ...",
"CNC Operator", // queryRole differs
results, hits, 0, nil,
)
if injected != 0 {
t.Errorf("expected 0 cross-role injections, got %d", injected)
}
if len(out) != 1 {
t.Errorf("results should be unchanged when gate rejects, got len=%d", len(out))
}
}
func TestInjectPlaybookMisses_RoleGateAllowsSameRole(t *testing.T) {
results := []Result{{ID: "e-5617", Corpus: "ethereal_workers", Distance: 0.50}}
hits := []PlaybookHit{{
PlaybookID: "pb-q2",
Distance: 0.135,
Entry: PlaybookEntry{
QueryText: "Need 1 Forklift Operator in Detroit MI ...",
Role: "Forklift Operator",
AnswerID: "e-6193",
AnswerCorpus: "ethereal_workers",
Score: 1.0,
},
}}
out, injected := InjectPlaybookMisses(
"Need 2 Forklift Operators in Detroit MI ...",
"Forklift Operators", // plural — roleEqual normalizes to match
results, hits, 0, nil,
)
if injected != 1 {
t.Errorf("expected 1 same-role injection, got %d", injected)
}
if len(out) != 2 {
t.Errorf("results should be extended with the injection, got len=%d", len(out))
}
}
func TestInjectPlaybookMisses_RoleGateBackwardCompat(t *testing.T) {
// Empty role on either side disables the gate — preserves
// behavior for callers (lift suite, etc.) that don't supply role.
results := []Result{{ID: "w-3759", Corpus: "workers", Distance: 0.50}}
hits := []PlaybookHit{{
PlaybookID: "pb-q2",
Distance: 0.135,
Entry: PlaybookEntry{
QueryText: "...",
Role: "Forklift Operator", // entry has role
AnswerID: "e-6193",
AnswerCorpus: "ethereal_workers",
Score: 1.0,
},
}}
// Caller didn't pass queryRole — gate must NOT fire.
_, injected := InjectPlaybookMisses("...", "", results, hits, 0, nil)
if injected != 1 {
t.Errorf("expected gate disabled when queryRole empty (injected=1), got %d", injected)
}
// Inverse: queryRole set but entry has no role.
hits[0].Entry.Role = ""
_, injected = InjectPlaybookMisses("...", "Forklift Operator", results, hits, 0, nil)
if injected != 1 {
t.Errorf("expected gate disabled when entry.Role empty (injected=1), got %d", injected)
}
}
func TestApplyPlaybookBoost_RoleGateRejectsCrossRole(t *testing.T) {
results := []Result{
{ID: "w-3759", Corpus: "workers", Distance: 0.50},
{ID: "e-6193", Corpus: "ethereal_workers", Distance: 0.55},
}
hits := []PlaybookHit{{
PlaybookID: "pb-q2",
Distance: 0.18,
Entry: PlaybookEntry{
QueryText: "Forklift Operator query",
Role: "Forklift Operator",
AnswerID: "e-6193",
AnswerCorpus: "ethereal_workers",
Score: 1.0,
},
}}
// Different-role query → boost must NOT fire on e-6193 even though
// it's in the result set.
n := ApplyPlaybookBoost("CNC Operator", results, hits)
if n != 0 {
t.Errorf("expected 0 cross-role boosts, got %d", n)
}
if results[1].Distance != 0.55 {
t.Errorf("e-6193 distance should be unchanged (0.55), got %v", results[1].Distance)
}
}
func TestRoleEqual_PluralAndCase(t *testing.T) {
cases := []struct {
a, b string
want bool
}{
{"Forklift Operator", "forklift operator", true},
{"Forklift Operator", "Forklift Operators", true}, // plural
{"CNC Operator", "Forklift Operator", false},
{"Pickers", "Picker", true},
{"", "Forklift Operator", true}, // gate disabled
{"CNC Operator", "", true}, // gate disabled (other side empty)
{"", "", true}, // both empty = both disabled
{"Boxes", "Box", true}, // -es plural
}
for _, c := range cases {
got := roleEqual(c.a, c.b)
if got != c.want {
t.Errorf("roleEqual(%q, %q) = %v, want %v", c.a, c.b, got, c.want)
}
}
}
func abs(f float64) float64 {
if f < 0 {
return -f