reality_test real_003: 40-query paraphrase stress + extractor extension
Stress-tests the role gate with 40 queries (10 fill_events rows × 4
styles): need, client_first, looking, shorthand. Each row's role +
client + city stays the same; only the surface phrasing changes.
real_003 (original extractor) confirmed the shorthand-vs-shorthand
failure mode: CNC Operator shorthand recording leaked w-2404 onto
Forklift Operator shorthand query within the same Beacon Freight
Detroit cluster. Both record + query had empty role (extractor
returns "" for shorthand because there's no separator between role
and city), gate disabled, distance check passed, bleed fired.
Fix: extended extractRoleFromNeed to handle client_first
("{client} needs N {role} in...") and looking ("Looking for N
{role} at...") patterns. Shorthand left intentionally unmatched —
"Forklift Operator Detroit" is shape-indistinguishable from
"Forklift" + "Operator Detroit" without an LLM extractor or known-
cities lookup.
real_003b (extended extractor) verifies bleed closed across all 4
styles for this dataset. Forklift Operator queries keep w-2136 (the
cold-pass-correct match) regardless of which style the query came
in. Same-role boosts now fire correctly across styles — a CNC
Operator recording made in `looking` style boosts the CNC need-form
query.
scripts/cutover/gen_real_queries.go: added -styles flag with values
need|client_first|looking|shorthand|all (default need preserves
real_001/002 behavior). Tests/reality/real_coord_queries_v2.txt is
the 40-query stress file.
scripts/playbook_lift/main_test.go: 10 sub-tests lock the four
documented patterns + shorthand limitation + lift-suite-style
queries (no clean role, returns empty as expected).
Aggregate metrics:
- real_003 (original): disc=7, lift=7, boost=14, meanΔ=-0.108
- real_003b (extended): disc=11, lift=10, boost=31, meanΔ=-0.202
The growth reflects more LEGITIMATE same-role same-cluster transfer
firing across styles, not bleed (verified by per-cluster bleed
table — Forklift Operator queries unchanged across all 4 styles).
Known limitation documented in real_003_findings.md: same-cluster,
same-role queries in shorthand still embed close enough that a
shorthand recording could bleed onto a different-role shorthand
query if both record + query strip role. Closing this requires
LLM extraction or known-cities lookup at record + query time.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -19,6 +19,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
"github.com/apache/arrow-go/v18/parquet/file"
|
||||
@@ -27,7 +28,8 @@ import (
|
||||
|
||||
func main() {
|
||||
src := flag.String("src", "/home/profit/lakehouse/data/datasets/fill_events.parquet", "fill_events parquet path")
|
||||
limit := flag.Int("limit", 10, "number of queries to generate")
|
||||
limit := flag.Int("limit", 10, "number of source rows to read")
|
||||
styles := flag.String("styles", "need", "comma-separated styles to emit per row (need|client_first|looking|shorthand|all)")
|
||||
flag.Parse()
|
||||
|
||||
r, err := file.OpenParquetFile(*src, false)
|
||||
@@ -61,35 +63,98 @@ func main() {
|
||||
n = *limit
|
||||
}
|
||||
|
||||
stylesList := parseStyles(*styles)
|
||||
|
||||
fmt.Println("# Real-shape coordinator queries — generated from fill_events.parquet")
|
||||
fmt.Println("# (real-shape demand data; queries built mechanically from event rows).")
|
||||
fmt.Printf("# Source: %s (%d rows total, %d emitted)\n", *src, tbl.NumRows(), n)
|
||||
fmt.Printf("# Source: %s (%d rows total, %d emitted, styles=%v)\n", *src, tbl.NumRows(), n, stylesList)
|
||||
fmt.Println("#")
|
||||
fmt.Println("# Format: client + count + role + city/state + start time +")
|
||||
fmt.Println("# (optional deadline). Mimics the natural language a coordinator would")
|
||||
fmt.Println("# type into a dispatch tool when triaging the next-up demand.")
|
||||
fmt.Println("# Styles:")
|
||||
fmt.Println("# need: 'Need N {role}{s} in {city} {state} starting at {at} for {client}'")
|
||||
fmt.Println("# — matches scripts/playbook_lift's extractRoleFromNeed regex")
|
||||
fmt.Println("# client_first: '{client} needs N {role}{s} in {city} {state} at {at}'")
|
||||
fmt.Println("# looking: 'Looking for N {role}{s} at {client} in {city} {state} for {at} shift'")
|
||||
fmt.Println("# shorthand: 'N {role}{s} {city} {state} {at} {client}'")
|
||||
fmt.Println("#")
|
||||
fmt.Println("# Only 'need' currently extracts a role. The other three test the")
|
||||
fmt.Println("# substrate's bleed behavior when the role gate is silently disabled.")
|
||||
fmt.Println()
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
c := client.ValueStr(i)
|
||||
cy := city.ValueStr(i)
|
||||
st := state.ValueStr(i)
|
||||
ro := role.ValueStr(i)
|
||||
ct := count.ValueStr(i)
|
||||
t := at.ValueStr(i)
|
||||
dl := deadline.ValueStr(i)
|
||||
|
||||
// Phrase one is the urgent ask; phrase two is the natural rephrase
|
||||
// a coordinator might use when typing fast. Different syntax,
|
||||
// same intent — exercises the embedder's paraphrase tolerance.
|
||||
q := fmt.Sprintf("Need %s %s in %s %s starting at %s for %s", ct, pluralize(ro, ct), cy, st, t, c)
|
||||
if dl != "" && dl != "(null)" {
|
||||
q += fmt.Sprintf(", deadline %s", dl)
|
||||
ev := event{
|
||||
client: client.ValueStr(i),
|
||||
city: city.ValueStr(i),
|
||||
state: state.ValueStr(i),
|
||||
role: role.ValueStr(i),
|
||||
count: count.ValueStr(i),
|
||||
at: at.ValueStr(i),
|
||||
}
|
||||
if dl := deadline.ValueStr(i); dl != "" && dl != "(null)" {
|
||||
ev.deadline = dl
|
||||
}
|
||||
for _, s := range stylesList {
|
||||
fmt.Println(formatQuery(ev, s))
|
||||
}
|
||||
fmt.Println(q)
|
||||
}
|
||||
}
|
||||
|
||||
type event struct {
|
||||
client, city, state, role, count, at, deadline string
|
||||
}
|
||||
|
||||
func formatQuery(e event, style string) string {
|
||||
r := pluralize(e.role, e.count)
|
||||
switch style {
|
||||
case "client_first":
|
||||
// No "Need ... in" anchor — extractRoleFromNeed returns "" on this.
|
||||
return fmt.Sprintf("%s needs %s %s in %s %s at %s", e.client, e.count, r, e.city, e.state, e.at)
|
||||
case "looking":
|
||||
return fmt.Sprintf("Looking for %s %s at %s in %s %s for %s shift", e.count, r, e.client, e.city, e.state, e.at)
|
||||
case "shorthand":
|
||||
return fmt.Sprintf("%s %s %s %s %s %s", e.count, r, e.city, e.state, e.at, e.client)
|
||||
default:
|
||||
// "need" form — the original real_001 shape, regex-extractor wins.
|
||||
q := fmt.Sprintf("Need %s %s in %s %s starting at %s for %s", e.count, r, e.city, e.state, e.at, e.client)
|
||||
if e.deadline != "" {
|
||||
q += ", deadline " + e.deadline
|
||||
}
|
||||
return q
|
||||
}
|
||||
}
|
||||
|
||||
// parseStyles unpacks the comma-separated -styles flag, with "all"
|
||||
// expanding to every supported style and unknown tokens dropped
|
||||
// (with a log line so callers know).
|
||||
func parseStyles(csv string) []string {
|
||||
all := []string{"need", "client_first", "looking", "shorthand"}
|
||||
if strings.TrimSpace(csv) == "all" {
|
||||
return all
|
||||
}
|
||||
out := []string{}
|
||||
for _, s := range strings.Split(csv, ",") {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
known := false
|
||||
for _, a := range all {
|
||||
if a == s {
|
||||
known = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
log.Printf("gen_real_queries: unknown style %q — skipping", s)
|
||||
continue
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{"need"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pluralize(role, count string) string {
|
||||
if count == "1" {
|
||||
return role
|
||||
|
||||
@@ -631,25 +631,53 @@ func appendNote(existing, add string) string {
|
||||
// refactor; harmless for now.
|
||||
var _ = sort.Slice
|
||||
|
||||
// extractRoleFromNeed pulls the role out of "Need N {role}{s} in {city}"
|
||||
// shape queries — the fill_events-derived form used by real_NNN runs.
|
||||
// Returns "" for any query that doesn't match (free-form lift-suite
|
||||
// queries fall back to empty, leaving the cross-role gate disabled).
|
||||
// extractRoleFromNeed pulls the role out of staffing-shape queries.
|
||||
// Returns "" for any query that doesn't match a known anchor pattern
|
||||
// (free-form lift-suite queries + shorthand-style fall back to empty,
|
||||
// leaving the cross-role gate disabled).
|
||||
//
|
||||
// The pattern is permissive: the count can be any digits, and the
|
||||
// role is everything between the count and " in ". This catches
|
||||
// "Need 5 Warehouse Associates in Kansas City" → "Warehouse Associates";
|
||||
// roleEqual on the matrix side handles plurals + case.
|
||||
// Patterns covered (in priority order):
|
||||
// need: "Need N {role}{s} in {city} ..."
|
||||
// client_first: "{client} needs N {role}{s} in {city} ..."
|
||||
// looking: "Looking for N {role}{s} at {client} in {city} ..."
|
||||
//
|
||||
// Pattern explicitly NOT covered:
|
||||
// shorthand: "N {role}{s} {city} {state} {at} {client}"
|
||||
// Because there's no separator between role and city in shorthand
|
||||
// ("Forklift Operator Detroit" is shape-indistinguishable from
|
||||
// "Forklift" + "Operator Detroit"), a regex can't reliably extract
|
||||
// role here. real_003 confirmed shorthand-vs-shorthand cross-role
|
||||
// bleed: a CNC Operator shorthand recording leaked w-2404 onto a
|
||||
// Forklift Operator shorthand query within the same Beacon Freight
|
||||
// Detroit cluster. Closing that requires either an LLM extractor at
|
||||
// record+query time or a known-cities lookup table.
|
||||
//
|
||||
// Lives here (not in internal/matrix) because role extraction from
|
||||
// free-form text is a caller concern; matrix only consumes the
|
||||
// already-resolved Role string. A future LLM-based extractor would
|
||||
// replace this regex without changing matrix's gate logic.
|
||||
// replace this function without changing matrix's gate logic.
|
||||
func extractRoleFromNeed(query string) string {
|
||||
re := regexp.MustCompile(`(?i)^Need\s+\d+\s+(.+?)\s+in\s+`)
|
||||
m := re.FindStringSubmatch(query)
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
for _, re := range roleExtractRegexes {
|
||||
if m := re.FindStringSubmatch(query); len(m) >= 2 {
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(m[1])
|
||||
return ""
|
||||
}
|
||||
|
||||
// roleExtractRegexes is ordered: more-specific anchors first so a
|
||||
// "Looking for ..." query doesn't accidentally land in the "Need"
|
||||
// pattern (impossible given the prefix, but guards against future
|
||||
// pattern additions). Compiled once at package init via MustCompile.
|
||||
var roleExtractRegexes = []*regexp.Regexp{
|
||||
// "Need N {role} in ..." — the original real_001 form.
|
||||
regexp.MustCompile(`(?i)^Need\s+\d+\s+(.+?)\s+in\s+`),
|
||||
// "Looking for N {role} at ..." — the looking style. Anchor on
|
||||
// "at" because the role is followed by client (preceded by "at"),
|
||||
// not by city directly.
|
||||
regexp.MustCompile(`(?i)^Looking\s+for\s+\d+\s+(.+?)\s+at\s+`),
|
||||
// "{client} needs N {role} in ..." — the client_first style.
|
||||
// Greedy on the client side via .+?, then "needs", then count,
|
||||
// then role, then "in".
|
||||
regexp.MustCompile(`(?i)^.+?\s+needs\s+\d+\s+(.+?)\s+in\s+`),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestExtractRoleFromNeed locks the four query-shape patterns documented
|
||||
// in real_003_findings.md so a future change to the regex can't silently
|
||||
// drop coverage of any production-shape style. Real_001 used `need`-only;
|
||||
// real_003 confirmed `shorthand` cross-role bleed; the extended
|
||||
// extractor in real_003b covers `client_first` + `looking` and leaves
|
||||
// `shorthand` as a known limitation (no separator between role and city).
|
||||
func TestExtractRoleFromNeed(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
query string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"need style — original real_001 form",
|
||||
"Need 1 Forklift Operator in Detroit MI starting at 15:00 for Beacon Freight",
|
||||
"Forklift Operator",
|
||||
},
|
||||
{
|
||||
"need with deadline trailer",
|
||||
"Need 4 Pickers in Detroit MI starting at 13:30 for Beacon Freight, deadline 2026-05-28",
|
||||
"Pickers",
|
||||
},
|
||||
{
|
||||
"client_first style — added in real_003b",
|
||||
"Beacon Freight needs 1 Forklift Operator in Detroit MI at 15:00",
|
||||
"Forklift Operator",
|
||||
},
|
||||
{
|
||||
"client_first with multi-word client",
|
||||
"Parallel Machining needs 5 Warehouse Associates in Kansas City MO at 09:00",
|
||||
"Warehouse Associates",
|
||||
},
|
||||
{
|
||||
"looking style — added in real_003b",
|
||||
"Looking for 1 Forklift Operator at Beacon Freight in Detroit MI for 15:00 shift",
|
||||
"Forklift Operator",
|
||||
},
|
||||
{
|
||||
"looking with multi-word role + 4-digit count",
|
||||
"Looking for 1234 Senior Production Supervisors at Heritage Foods in Flint MI for 08:30 shift",
|
||||
"Senior Production Supervisors",
|
||||
},
|
||||
{
|
||||
"shorthand — known limitation, returns empty",
|
||||
"1 Forklift Operator Detroit MI 15:00 Beacon Freight",
|
||||
"",
|
||||
},
|
||||
{
|
||||
"shorthand multi-word city — also empty",
|
||||
"5 Warehouse Associates Kansas City MO 09:00 Parallel Machining",
|
||||
"",
|
||||
},
|
||||
{
|
||||
"lift-suite multi-constraint — no clean role, returns empty",
|
||||
"Forklift operator with OSHA-30, warehouse experience, day shift availability",
|
||||
"",
|
||||
},
|
||||
{
|
||||
"OOD honesty signal — lift-suite, returns empty",
|
||||
"Dental hygienist with three years experience, Indianapolis area",
|
||||
"",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := extractRoleFromNeed(c.query)
|
||||
if got != c.want {
|
||||
t.Errorf("extractRoleFromNeed(%q) = %q, want %q", c.query, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user