reality_test real_003: 40-query paraphrase stress + extractor extension

Stress-tests the role gate with 40 queries (10 fill_events rows × 4
styles): need, client_first, looking, shorthand. Each row's role +
client + city stays the same; only the surface phrasing changes.

real_003 (original extractor) confirmed the shorthand-vs-shorthand
failure mode: CNC Operator shorthand recording leaked w-2404 onto
Forklift Operator shorthand query within the same Beacon Freight
Detroit cluster. Both record + query had empty role (extractor
returns "" for shorthand because there's no separator between role
and city), gate disabled, distance check passed, bleed fired.

Fix: extended extractRoleFromNeed to handle client_first
("{client} needs N {role} in...") and looking ("Looking for N
{role} at...") patterns. Shorthand left intentionally unmatched —
"Forklift Operator Detroit" is shape-indistinguishable from
"Forklift" + "Operator Detroit" without an LLM extractor or known-
cities lookup.

real_003b (extended extractor) verifies bleed closed across all 4
styles for this dataset. Forklift Operator queries keep w-2136 (the
cold-pass-correct match) regardless of which style the query came
in. Same-role boosts now fire correctly across styles — a CNC
Operator recording made in `looking` style boosts the CNC need-form
query.

scripts/cutover/gen_real_queries.go: added -styles flag with values
need|client_first|looking|shorthand|all (default need preserves
real_001/002 behavior). Tests/reality/real_coord_queries_v2.txt is
the 40-query stress file.

scripts/playbook_lift/main_test.go: 10 sub-tests lock the four
documented patterns + shorthand limitation + lift-suite-style
queries (no clean role, returns empty as expected).

Aggregate metrics:
- real_003  (original): disc=7,  lift=7,  boost=14, meanΔ=-0.108
- real_003b (extended): disc=11, lift=10, boost=31, meanΔ=-0.202
The growth reflects more LEGITIMATE same-role same-cluster transfer
firing across styles, not bleed (verified by per-cluster bleed
table — Forklift Operator queries unchanged across all 4 styles).

Known limitation documented in real_003_findings.md: same-cluster,
same-role queries in shorthand still embed close enough that a
shorthand recording could bleed onto a different-role shorthand
query if both record + query strip role. Closing this requires
LLM extraction or known-cities lookup at record + query time.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-30 21:42:02 -05:00
co-authored by Claude Opus 4.7
parent 997527be4d
commit 3263254f1c
8 changed files with 616 additions and 34 deletions
+85 -20
View File
@@ -19,6 +19,7 @@ import (
"flag"
"fmt"
"log"
"strings"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/apache/arrow-go/v18/parquet/file"
@@ -27,7 +28,8 @@ import (
func main() {
src := flag.String("src", "/home/profit/lakehouse/data/datasets/fill_events.parquet", "fill_events parquet path")
limit := flag.Int("limit", 10, "number of queries to generate")
limit := flag.Int("limit", 10, "number of source rows to read")
styles := flag.String("styles", "need", "comma-separated styles to emit per row (need|client_first|looking|shorthand|all)")
flag.Parse()
r, err := file.OpenParquetFile(*src, false)
@@ -61,35 +63,98 @@ func main() {
n = *limit
}
stylesList := parseStyles(*styles)
fmt.Println("# Real-shape coordinator queries — generated from fill_events.parquet")
fmt.Println("# (real-shape demand data; queries built mechanically from event rows).")
fmt.Printf("# Source: %s (%d rows total, %d emitted)\n", *src, tbl.NumRows(), n)
fmt.Printf("# Source: %s (%d rows total, %d emitted, styles=%v)\n", *src, tbl.NumRows(), n, stylesList)
fmt.Println("#")
fmt.Println("# Format: client + count + role + city/state + start time +")
fmt.Println("# (optional deadline). Mimics the natural language a coordinator would")
fmt.Println("# type into a dispatch tool when triaging the next-up demand.")
fmt.Println("# Styles:")
fmt.Println("# need: 'Need N {role}{s} in {city} {state} starting at {at} for {client}'")
fmt.Println("# — matches scripts/playbook_lift's extractRoleFromNeed regex")
fmt.Println("# client_first: '{client} needs N {role}{s} in {city} {state} at {at}'")
fmt.Println("# looking: 'Looking for N {role}{s} at {client} in {city} {state} for {at} shift'")
fmt.Println("# shorthand: 'N {role}{s} {city} {state} {at} {client}'")
fmt.Println("#")
fmt.Println("# Only 'need' currently extracts a role. The other three test the")
fmt.Println("# substrate's bleed behavior when the role gate is silently disabled.")
fmt.Println()
for i := 0; i < n; i++ {
c := client.ValueStr(i)
cy := city.ValueStr(i)
st := state.ValueStr(i)
ro := role.ValueStr(i)
ct := count.ValueStr(i)
t := at.ValueStr(i)
dl := deadline.ValueStr(i)
// Phrase one is the urgent ask; phrase two is the natural rephrase
// a coordinator might use when typing fast. Different syntax,
// same intent — exercises the embedder's paraphrase tolerance.
q := fmt.Sprintf("Need %s %s in %s %s starting at %s for %s", ct, pluralize(ro, ct), cy, st, t, c)
if dl != "" && dl != "(null)" {
q += fmt.Sprintf(", deadline %s", dl)
ev := event{
client: client.ValueStr(i),
city: city.ValueStr(i),
state: state.ValueStr(i),
role: role.ValueStr(i),
count: count.ValueStr(i),
at: at.ValueStr(i),
}
if dl := deadline.ValueStr(i); dl != "" && dl != "(null)" {
ev.deadline = dl
}
for _, s := range stylesList {
fmt.Println(formatQuery(ev, s))
}
fmt.Println(q)
}
}
type event struct {
client, city, state, role, count, at, deadline string
}
func formatQuery(e event, style string) string {
r := pluralize(e.role, e.count)
switch style {
case "client_first":
// No "Need ... in" anchor — extractRoleFromNeed returns "" on this.
return fmt.Sprintf("%s needs %s %s in %s %s at %s", e.client, e.count, r, e.city, e.state, e.at)
case "looking":
return fmt.Sprintf("Looking for %s %s at %s in %s %s for %s shift", e.count, r, e.client, e.city, e.state, e.at)
case "shorthand":
return fmt.Sprintf("%s %s %s %s %s %s", e.count, r, e.city, e.state, e.at, e.client)
default:
// "need" form — the original real_001 shape, regex-extractor wins.
q := fmt.Sprintf("Need %s %s in %s %s starting at %s for %s", e.count, r, e.city, e.state, e.at, e.client)
if e.deadline != "" {
q += ", deadline " + e.deadline
}
return q
}
}
// parseStyles unpacks the comma-separated -styles flag, with "all"
// expanding to every supported style and unknown tokens dropped
// (with a log line so callers know).
func parseStyles(csv string) []string {
all := []string{"need", "client_first", "looking", "shorthand"}
if strings.TrimSpace(csv) == "all" {
return all
}
out := []string{}
for _, s := range strings.Split(csv, ",") {
s = strings.TrimSpace(s)
if s == "" {
continue
}
known := false
for _, a := range all {
if a == s {
known = true
break
}
}
if !known {
log.Printf("gen_real_queries: unknown style %q — skipping", s)
continue
}
out = append(out, s)
}
if len(out) == 0 {
return []string{"need"}
}
return out
}
func pluralize(role, count string) string {
if count == "1" {
return role