playbook_lift: LLM-based role extractor closes shorthand bleed (real_004)

real_003 left a known-weak hole: shorthand-style queries
("{count} {role} {city} {state} ...") have no separator between
role and city, so a regex can't reliably extract — leaving the
cross-role gate disabled when both record AND query are shorthand.

This commit adds a roleExtractor with regex-first + LLM fallback:

- Regex first (fast, deterministic) — handles need + client_first +
  looking from real_003b. ~75% of styles, no LLM cost paid.
- LLM fallback when regex returns empty AND model is configured —
  Ollama-shape /api/chat with format=json, schema-tight prompt,
  temperature 0. ~1-3s on local qwen2.5.
- Per-process cache — paraphrase + rejudge passes reuse the same
  query 4× per run; cache prevents 4× LLM cost.
- Off-by-default — opt-in via -llm-role-extract flag (CLI) and
  LLM_ROLE_EXTRACT=1 env var (harness wrapper). real_003b shipping
  config unchanged unless explicitly enabled.

8 new tests in scripts/playbook_lift/main_test.go:
- TestRoleExtractor_RegexFirst: LLM not called when regex matches
- TestRoleExtractor_LLMFallback: shorthand goes to LLM
- TestRoleExtractor_LLMOffLeavesEmpty: opt-in default preserved
- TestRoleExtractor_Cache: 3 calls = 1 LLM hit
- TestRoleExtractor_NilSafe: nil receiver runs regex only
- TestExtractRoleViaLLM_HTTPError + _BadJSON: failure paths
- TestRoleExtractor_ClosesCrossRoleShorthandBleed: synthetic
  witness for the real_003 scenario — both record + query are
  shorthand, regex returns "" for both, LLM produces DIFFERENT
  role tokens for CNC vs Forklift, so matrix gate's cross-role
  rejection (locked separately in
  TestInjectPlaybookMisses_RoleGateRejectsCrossRole) fires
  correctly. This is the load-bearing verification.

Reality test real_004 ran the same 40-query stress as real_003 with
LLM extraction on. Cross-style same-role boosts fired correctly
across all 4 styles for Loaders + Packers + Shipping Clerk clusters
(including shorthand → other-style transfer). No cross-role bleed
observed. The reality test alone can't be a clean "with vs without"
comparison (HNSW build is non-deterministic across runs, and
real_004 stochastics didn't trigger a shorthand recording at all),
which is why the unit-test witness exists.

Production note (in real_004_findings.md): LLM extraction is for
reality-test coverage of arbitrary query shapes. Production should
extract role at INGEST time (when the inbox parser already runs an
LLM) and pass already-resolved role through requests — same shape
as multi_coord_stress's existing Demand{Role: ...} model. The hot
path should never need the harness extractor's per-query LLM cost.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

scripts/playbook_lift.sh

@@ -283,6 +283,16 @@ fi
 if [ "$WITH_REJUDGE" = "1" ]; then
   EXTRA_FLAGS="$EXTRA_FLAGS -with-rejudge"
 fi
+# LLM_ROLE_EXTRACT=1 enables the qwen2.5 fallback extractor for
+# shorthand-style queries (real_003 known limitation). Adds ~1-3s
+# per shorthand query but closes the cross-role bleed at the
+# extraction layer. Default off.
+if [ "${LLM_ROLE_EXTRACT:-0}" = "1" ]; then
+  EXTRA_FLAGS="$EXTRA_FLAGS -llm-role-extract"
+  if [ -n "${LLM_ROLE_MODEL:-}" ]; then
+    EXTRA_FLAGS="$EXTRA_FLAGS -llm-role-model $LLM_ROLE_MODEL"
+  fi
+fi
 ./bin/playbook_lift \
   -config  "$CONFIG_PATH" \
   -gateway "http://127.0.0.1:3110" \

scripts/playbook_lift/main.go

@@ -148,6 +148,8 @@ func main() {
 	out := flag.String("out", "reports/reality-tests/playbook_lift_001.json", "output JSONL path")
 	withParaphrase := flag.Bool("with-paraphrase", false, "after warm pass, generate a paraphrase via the judge model and re-query with playbook=true to test the learning property")
 	withRejudge := flag.Bool("with-rejudge", false, "after warm pass, judge warm top-1 to measure QUALITY lift (vs cold top-1 rating), not just rank-of-cold-judge-best")
+	llmRoleExtract := flag.Bool("llm-role-extract", false, "fall back to LLM (qwen2.5 format=json) when the regex extractor returns empty — closes the shorthand-style cross-role bleed surfaced in real_003 at the cost of ~1-3s/query")
+	llmRoleModel := flag.String("llm-role-model", "qwen2.5:latest", "Ollama model used for LLM role extraction; ignored when -llm-role-extract is off")
 	flag.Parse()
 
 	// Judge resolution priority: explicit flag > $JUDGE_MODEL env >
@@ -176,6 +178,23 @@ func main() {
 	log.Printf("[lift] %d queries · corpora=%v · k=%d · judge=%s", len(qs), corpora, *k, *judge)
 
 	hc := &http.Client{Timeout: 60 * time.Second}
+
+	// Package-global role extractor — used by matrixSearch +
+	// playbookRecord. Off-by-default so the existing harness behavior
+	// (regex-only extraction) is unchanged unless -llm-role-extract.
+	{
+		mdl := ""
+		if *llmRoleExtract {
+			mdl = *llmRoleModel
+			log.Printf("[lift] llm role extraction ON (model=%s) — shorthand queries get LLM fallback", mdl)
+		}
+		globalRoleExtractor = &roleExtractor{
+			hc:        hc,
+			ollamaURL: *ollama,
+			model:     mdl,
+		}
+	}
+
 	runs := make([]queryRun, 0, len(qs))
 	totalDelta := float32(0)
 	playbookBoostedTotal := 0
@@ -482,11 +501,12 @@ func matrixSearch(hc *http.Client, gw, query string, corpora []string, k int, us
 		"per_corpus_k": k,
 		"use_playbook": usePlaybook,
 	}
-	// Mechanical role extraction (real_001 cross-role bleed fix).
-	// Empty when the query doesn't match the "Need N {role}{s} in"
-	// pattern — gate stays disabled and the harness behaves as before
-	// for the lift suite's free-form queries.
-	if role := extractRoleFromNeed(query); role != "" {
+	// Role extraction (real_001 + real_003 cross-role bleed fixes).
+	// Goes through globalRoleExtractor so shorthand-style queries get
+	// LLM fallback when -llm-role-extract is on. Empty result leaves
+	// the gate disabled — harness preserves current behavior on
+	// truly-unparseable shapes.
+	if role := globalRoleExtractor.extract(query); role != "" {
 		body["query_role"] = role
 	}
 	bs, _ := json.Marshal(body)
@@ -516,9 +536,10 @@ func playbookRecord(hc *http.Client, gw, query, answerID, answerCorpus string, s
 		"score":         score,
 		"tags":          []string{"reality-test", "playbook-lift-001"},
 	}
-	// Same mechanical role extraction as matrixSearch — recorded role
-	// lets retrieve-time gate fire on cross-role queries (real_001 fix).
-	if role := extractRoleFromNeed(query); role != "" {
+	// Same extractor as matrixSearch — shared cache, same LLM fallback
+	// rules. Recorded role lets retrieve-time gate fire on cross-role
+	// queries (real_001 + real_003 fixes).
+	if role := globalRoleExtractor.extract(query); role != "" {
 		body["role"] = role
 	}
 	bs, _ := json.Marshal(body)
@@ -631,6 +652,110 @@ func appendNote(existing, add string) string {
 // refactor; harmless for now.
 var _ = sort.Slice
 
+// globalRoleExtractor is set in main() and read by matrixSearch +
+// playbookRecord. nil-safe: a nil receiver on roleExtractor.extract
+// degrades to regex-only behavior, so the field is checked once at
+// startup and never re-validated per call.
+var globalRoleExtractor *roleExtractor
+
+// roleExtractor combines the fast-path regex with an optional LLM
+// fallback so callers can pay for shorthand coverage only when they
+// need it. Per real_003_findings.md: shorthand-style queries
+// ("N {role} {city} {state} {at} {client}") have no separator
+// between role and city, so a regex can't reliably extract — but a
+// small LLM with a tight format=json prompt can. Cost is ~1-3s per
+// extraction on local qwen2.5; cached per-process so paraphrase
+// passes don't pay twice for the same query.
+//
+// Empty cache + off-by-default LLM = the existing real_003b behavior
+// is unchanged unless callers explicitly enable LLM mode.
+type roleExtractor struct {
+	hc        *http.Client
+	ollamaURL string
+	model     string // "" disables LLM fallback
+	cache     map[string]string
+}
+
+// extract returns the role for a query. Tries regex first (fast,
+// deterministic); if regex misses and LLM is configured, calls Ollama;
+// caches the result either way. Returns "" when both miss — the
+// caller's gate stays disabled, preserving current behavior on
+// truly-unparseable shapes.
+func (r *roleExtractor) extract(query string) string {
+	if r == nil {
+		return extractRoleFromNeed(query)
+	}
+	if cached, ok := r.cache[query]; ok {
+		return cached
+	}
+	role := extractRoleFromNeed(query)
+	if role == "" && r.model != "" {
+		if v, err := extractRoleViaLLM(r.hc, r.ollamaURL, r.model, query); err == nil {
+			role = v
+		} else {
+			log.Printf("[lift] llm-role-extract failed (%v) — falling back to empty for %q", err, abbrev(query, 60))
+		}
+	}
+	if r.cache == nil {
+		r.cache = make(map[string]string)
+	}
+	r.cache[query] = role
+	return role
+}
+
+// extractRoleViaLLM asks the Ollama-shape /api/chat to identify the
+// staffing role in a free-form query. Tight schema + format=json so
+// parsing is deterministic. Empty role string is a valid response —
+// the model may decline to extract when the query has no clean role
+// (e.g. lift-suite multi-constraint queries).
+func extractRoleViaLLM(hc *http.Client, ollamaURL, model, query string) (string, error) {
+	system := `You are a staffing-domain role extractor.
+Output JSON only: {"role": "<role string or empty>"}.
+Rules:
+- Identify the staffing role (job title) the query is asking for.
+- Return only the role noun phrase — e.g. "Forklift Operator", "Pickers".
+- Preserve plurality from the query (don't singularize).
+- Strip qualifiers ("OSHA-30 certified") — we want the bare role.
+- If the query has no clean staffing role, return "" (empty string).
+- Do NOT explain. Just emit the JSON.`
+	body, _ := json.Marshal(map[string]any{
+		"model":  model,
+		"stream": false,
+		"format": "json",
+		"messages": []map[string]string{
+			{"role": "system", "content": system},
+			{"role": "user", "content": query},
+		},
+		"options": map[string]any{"temperature": 0.0},
+	})
+	req, _ := http.NewRequest("POST", ollamaURL+"/api/chat", bytes.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := hc.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		return "", fmt.Errorf("ollama chat: HTTP %d", resp.StatusCode)
+	}
+	rb, _ := io.ReadAll(resp.Body)
+	var ollamaResp struct {
+		Message struct {
+			Content string `json:"content"`
+		} `json:"message"`
+	}
+	if err := json.Unmarshal(rb, &ollamaResp); err != nil {
+		return "", err
+	}
+	var out struct {
+		Role string `json:"role"`
+	}
+	if err := json.Unmarshal([]byte(ollamaResp.Message.Content), &out); err != nil {
+		return "", fmt.Errorf("decode role: %w (content=%q)", err, ollamaResp.Message.Content)
+	}
+	return strings.TrimSpace(out.Role), nil
+}
+
 // extractRoleFromNeed pulls the role out of staffing-shape queries.
 // Returns "" for any query that doesn't match a known anchor pattern
 // (free-form lift-suite queries + shorthand-style fall back to empty,

scripts/playbook_lift/main_test.go

@@ -1,6 +1,14 @@
 package main
 
-import "testing"
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+)
 
 // TestExtractRoleFromNeed locks the four query-shape patterns documented
 // in real_003_findings.md so a future change to the regex can't silently
@@ -74,3 +82,202 @@ func TestExtractRoleFromNeed(t *testing.T) {
 		})
 	}
 }
+
+// fakeOllama returns a minimal Ollama-shape /api/chat handler that
+// echoes a fixed role string. Counts hits so caching tests can assert
+// on call count.
+func fakeOllama(role string) (*httptest.Server, *int64) {
+	var hits int64
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/chat" {
+			http.NotFound(w, r)
+			return
+		}
+		atomic.AddInt64(&hits, 1)
+		_, _ = io.ReadAll(r.Body)
+		// Ollama wraps the model's response under message.content as
+		// a JSON-string. The model's output is itself the
+		// {"role":"..."} JSON our extractor parses.
+		inner, _ := json.Marshal(map[string]string{"role": role})
+		out, _ := json.Marshal(map[string]any{
+			"message": map[string]string{"content": string(inner)},
+		})
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write(out)
+	}))
+	return srv, &hits
+}
+
+// TestRoleExtractor_RegexFirst locks the priority order: when the
+// regex matches, the LLM must NOT be called even if configured. Same
+// behavior the real_003b shipping config relies on (don't pay LLM
+// cost on need/client_first/looking queries).
+func TestRoleExtractor_RegexFirst(t *testing.T) {
+	srv, hits := fakeOllama("LLM-WONT-BE-CALLED")
+	defer srv.Close()
+	rx := &roleExtractor{
+		hc:        srv.Client(),
+		ollamaURL: srv.URL,
+		model:     "test-model", // LLM is configured...
+	}
+	got := rx.extract("Need 5 Forklift Operators in Detroit MI starting at 09:00 for ACME")
+	if got != "Forklift Operators" {
+		t.Errorf("regex should win on Need-form query, got %q", got)
+	}
+	if *hits != 0 {
+		t.Errorf("LLM should not be called when regex matched, got %d hits", *hits)
+	}
+}
+
+// TestRoleExtractor_LLMFallback locks the shorthand-coverage path:
+// when regex returns empty AND LLM is configured, the LLM is called
+// and its result is used. Closes the real_003 shorthand bleed at the
+// extraction layer.
+func TestRoleExtractor_LLMFallback(t *testing.T) {
+	srv, hits := fakeOllama("CNC Operator")
+	defer srv.Close()
+	rx := &roleExtractor{
+		hc:        srv.Client(),
+		ollamaURL: srv.URL,
+		model:     "test-model",
+	}
+	got := rx.extract("1 CNC Operator Detroit MI 17:30 Beacon Freight")
+	if got != "CNC Operator" {
+		t.Errorf("LLM fallback should fire on shorthand, got %q", got)
+	}
+	if *hits != 1 {
+		t.Errorf("expected exactly 1 LLM hit, got %d", *hits)
+	}
+}
+
+// TestRoleExtractor_LLMOffLeavesEmpty locks the opt-in default: with
+// LLM model unset, shorthand queries return empty (preserves
+// real_003b shipping config — no LLM cost paid by default).
+func TestRoleExtractor_LLMOffLeavesEmpty(t *testing.T) {
+	rx := &roleExtractor{} // model = "" disables LLM
+	got := rx.extract("1 CNC Operator Detroit MI 17:30 Beacon Freight")
+	if got != "" {
+		t.Errorf("with LLM off, shorthand should return empty, got %q", got)
+	}
+}
+
+// TestRoleExtractor_Cache locks the per-process cache: re-extracting
+// the same query must NOT call the LLM twice. Paraphrase passes hit
+// the same query under cold + warm + paraphrase + rejudge — without
+// the cache that's 4× LLM cost per query.
+func TestRoleExtractor_Cache(t *testing.T) {
+	srv, hits := fakeOllama("Pickers")
+	defer srv.Close()
+	rx := &roleExtractor{
+		hc:        srv.Client(),
+		ollamaURL: srv.URL,
+		model:     "test-model",
+	}
+	q := "4 Pickers Detroit MI 13:30 Beacon Freight"
+	for i := 0; i < 3; i++ {
+		got := rx.extract(q)
+		if got != "Pickers" {
+			t.Errorf("call %d: got %q, want Pickers", i, got)
+		}
+	}
+	if *hits != 1 {
+		t.Errorf("expected 1 LLM hit (rest cached), got %d", *hits)
+	}
+}
+
+// TestRoleExtractor_NilSafe locks the nil-receiver behavior so call
+// sites in matrixSearch + playbookRecord don't need a guard. nil
+// extractor degrades to regex-only.
+func TestRoleExtractor_NilSafe(t *testing.T) {
+	var rx *roleExtractor
+	got := rx.extract("Need 1 Loader in Indianapolis IN starting at 12:00 for Midway")
+	if got != "Loader" {
+		t.Errorf("nil receiver should still run regex, got %q", got)
+	}
+	got = rx.extract("1 CNC Operator Detroit MI 17:30 Beacon")
+	if got != "" {
+		t.Errorf("nil receiver should return empty when regex misses (no LLM), got %q", got)
+	}
+}
+
+// TestExtractRoleViaLLM_HTTPError locks the failure path: HTTP non-2xx
+// surfaces as an error so the caller can fall back to empty cleanly.
+func TestExtractRoleViaLLM_HTTPError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "internal", http.StatusInternalServerError)
+	}))
+	defer srv.Close()
+	_, err := extractRoleViaLLM(srv.Client(), srv.URL, "test-model", "anything")
+	if err == nil || !strings.Contains(err.Error(), "HTTP 500") {
+		t.Errorf("expected HTTP 500 error, got %v", err)
+	}
+}
+
+// TestRoleExtractor_ClosesCrossRoleShorthandBleed is the synthetic
+// witness for the real_003 finding: when both record AND query are
+// shorthand-style (no anchor between role and city), the regex
+// returns "" for both. With LLM off, both sides go empty → gate
+// disabled → cross-role bleed possible. With LLM on, both sides
+// extract a non-empty role → role-mismatch is caught at the matrix
+// gate.
+//
+// This test isolates the EXTRACTION layer (does the harness produce
+// the right role token?). The matrix gate's behavior on those tokens
+// is locked separately in internal/matrix/playbook_test.go's
+// TestInjectPlaybookMisses_RoleGateRejectsCrossRole.
+func TestRoleExtractor_ClosesCrossRoleShorthandBleed(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		// Determine which role to return based on the user message.
+		role := ""
+		if strings.Contains(string(body), "CNC") {
+			role = "CNC Operator"
+		} else if strings.Contains(string(body), "Forklift") {
+			role = "Forklift Operator"
+		}
+		inner, _ := json.Marshal(map[string]string{"role": role})
+		out, _ := json.Marshal(map[string]any{
+			"message": map[string]string{"content": string(inner)},
+		})
+		_, _ = w.Write(out)
+	}))
+	defer srv.Close()
+
+	rx := &roleExtractor{
+		hc:        srv.Client(),
+		ollamaURL: srv.URL,
+		model:     "test-model",
+	}
+
+	// Both queries are shorthand — regex would return "" for both.
+	// LLM extracts a real role for each, and they differ.
+	cnc := rx.extract("1 CNC Operator Detroit MI 17:30 Beacon Freight")
+	fork := rx.extract("1 Forklift Operator Detroit MI 15:00 Beacon Freight")
+
+	if cnc != "CNC Operator" {
+		t.Errorf("CNC shorthand should extract via LLM, got %q", cnc)
+	}
+	if fork != "Forklift Operator" {
+		t.Errorf("Forklift shorthand should extract via LLM, got %q", fork)
+	}
+	if cnc == fork {
+		t.Errorf("cross-role shorthand should produce DIFFERENT role tokens (cnc=%q fork=%q)", cnc, fork)
+	}
+}
+
+// TestExtractRoleViaLLM_BadJSON locks the model-output validation: if
+// the LLM returns non-JSON content, the error is surfaced (don't
+// silently treat it as empty role).
+func TestExtractRoleViaLLM_BadJSON(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		out, _ := json.Marshal(map[string]any{
+			"message": map[string]string{"content": "this is not json"},
+		})
+		_, _ = w.Write(out)
+	}))
+	defer srv.Close()
+	_, err := extractRoleViaLLM(srv.Client(), srv.URL, "test-model", "anything")
+	if err == nil || !strings.Contains(err.Error(), "decode role") {
+		t.Errorf("expected decode error, got %v", err)
+	}
+}