scripts: chicago analyzer field-name fixes + vectorize sanitizer hardening

Two small fixes surfaced during smoke testing:

analyze_chicago_contracts.ts: permit field is contact_1_name not
contact_1; reported_cost is integer-string. Fixed filter (was rejecting
all 2853 permits) and contractor extraction (was empty).

vectorize_raw_corpus.ts: sanitize() expanded to strip control chars +
ALL backslashes (kills incomplete \uXXXX escapes) + UTF-16 surrogates
(unpaired surrogates from emoji split by truncate boundary). Llm_team
response cache had docs with all three pollution shapes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.mcp.json

@@ -7,6 +7,14 @@
         "LAKEHOUSE_URL": "http://localhost:3100",
         "MCP_TRANSPORT": "stdio"
       }
+    },
+    "gitea": {
+      "command": "bunx",
+      "args": ["gitea-mcp"],
+      "env": {
+        "GITEA_HOST": "https://git.agentview.dev",
+        "GITEA_ACCESS_TOKEN": "SET_ME_FROM_GITEA_UI_USER_SETTINGS_APPLICATIONS"
+      }
     }
   }
 }

Cargo.lock

@@ -6898,6 +6898,7 @@ dependencies = [
  "storaged",
  "tokio",
  "tracing",
+ "truth",
  "url",
 ]
 
@@ -8736,6 +8737,7 @@ dependencies = [
  "serde",
  "serde_json",
  "tokio",
+ "toml",
  "tracing",
 ]
 
@@ -8905,6 +8907,17 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "validator"
+version = "0.1.0"
+dependencies = [
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "valuable"
 version = "0.1.1"

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "crates/lance-bench",
     "crates/vectord-lance",
     "crates/truth",
+    "crates/validator",
 ]
 
 [workspace.dependencies]

config/providers.toml

@@ -0,0 +1,62 @@
+# Phase 39: Provider Registry
+#
+# Per-provider base_url, auth scheme, and default model. The gateway's
+# /v1/chat dispatcher reads this file at boot to populate its provider
+# table. Secrets (API keys) come from /etc/lakehouse/secrets.toml or
+# environment variables — NEVER inline a key here.
+#
+# Adding a new provider:
+#   1. New [[provider]] block with name, base_url, auth, default_model
+#   2. Matching adapter at crates/aibridge/src/providers/<name>.rs
+#      implementing the ProviderAdapter trait (chat + embed + unload)
+#   3. Route arm in crates/gateway/src/v1/mod.rs matching on `name`
+#   4. Model-prefix routing hint in resolve_provider() if the provider
+#      uses an "<name>/..." model prefix (e.g. "openrouter/...")
+
+[[provider]]
+name = "ollama"
+base_url = "http://localhost:3200"
+auth = "none"
+default_model = "qwen3.5:latest"
+# Hot-path local inference. No bearer needed — Python sidecar on
+# localhost handles the Ollama API. Model names are bare
+# (e.g. "qwen3.5:latest", not "ollama/qwen3.5:latest").
+
+[[provider]]
+name = "ollama_cloud"
+base_url = "https://ollama.com"
+auth = "bearer"
+auth_env = "OLLAMA_CLOUD_KEY"
+default_model = "gpt-oss:120b"
+# Cloud-tier Ollama. Key resolved from OLLAMA_CLOUD_KEY env at gateway
+# boot. Model-prefix routing: "cloud/<model>" auto-routes here
+# (see gateway::v1::resolve_provider).
+
+[[provider]]
+name = "openrouter"
+base_url = "https://openrouter.ai/api/v1"
+auth = "bearer"
+auth_env = "OPENROUTER_API_KEY"
+auth_fallback_files = ["/home/profit/.env", "/root/llm_team_config.json"]
+default_model = "openai/gpt-oss-120b:free"
+# Multi-provider gateway. Covers Anthropic, Google, OpenAI, MiniMax,
+# Qwen, Gemma, etc. Key resolved via crates/gateway/src/v1/openrouter.rs
+# resolve_openrouter_key() — env first, then fallback files.
+# Model-prefix routing: "openrouter/<vendor>/<model>" auto-routes here,
+# prefix stripped before upstream call.
+
+# Planned (Phase 40 long-horizon — adapters not yet shipped):
+#
+# [[provider]]
+# name = "gemini"
+# base_url = "https://generativelanguage.googleapis.com/v1beta"
+# auth = "api_key_query"
+# auth_env = "GEMINI_API_KEY"
+# default_model = "gemini-2.0-flash"
+#
+# [[provider]]
+# name = "claude"
+# base_url = "https://api.anthropic.com/v1"
+# auth = "x_api_key"
+# auth_env = "ANTHROPIC_API_KEY"
+# default_model = "claude-3-5-sonnet-latest"

crates/aibridge/src/context.rs

@@ -13,11 +13,9 @@
 use std::collections::HashMap;
 use std::sync::OnceLock;
 
-/// Rough token count. `chars / 4` ceiling. See module docs for why
+// `estimate_tokens` moved to `shared::model_matrix::ModelMatrix::estimate_tokens`
-/// this heuristic is sufficient.
+// (cdc24d8). All callers migrated; the deprecated wrapper that stood in its
-pub fn estimate_tokens(text: &str) -> usize {
+// place has been removed since it had zero external consumers.
-    (text.chars().count() + 3) / 4
-}
 
 /// Phase 21 — per-model context windows, mirroring the TS table in
 /// `tests/multi-agent/agent.ts`. Anchored on each model's documented
@@ -84,8 +82,8 @@ pub fn assert_context_budget(
     let window = context_window_for(model);
     let safety = opts.safety_margin.unwrap_or(DEFAULT_SAFETY_MARGIN);
     let max_tokens = opts.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS);
-    let sys_tokens = opts.system.map(estimate_tokens).unwrap_or(0);
+    let sys_tokens = opts.system.map(shared::model_matrix::ModelMatrix::estimate_tokens).unwrap_or(0);
-    let estimated = estimate_tokens(prompt) + sys_tokens + max_tokens;
+    let estimated = shared::model_matrix::ModelMatrix::estimate_tokens(prompt) + sys_tokens + max_tokens;
     let remaining = window as i64 - estimated as i64 - safety as i64;
     let check = BudgetCheck { estimated, window, remaining };
     if remaining < 0 && !opts.bypass {
@@ -109,14 +107,10 @@ pub fn overflow_message(model: &str, check: &BudgetCheck, over_by: usize, safety
 mod tests {
     use super::*;
 
-    #[test]
+    // Deprecated-function behavior is now canonically tested in
-    fn estimate_tokens_ceiling_divides_by_four() {
+    // crates/shared/src/model_matrix.rs. This test was the legacy
-        assert_eq!(estimate_tokens(""), 0);
+    // pin that preceded the migration; delete when the deprecated
-        assert_eq!(estimate_tokens("abc"), 1);    // 3 → ceil(3/4) = 1
+    // wrapper itself goes (see the #[deprecated] attribute).
-        assert_eq!(estimate_tokens("abcd"), 1);   // 4 → ceil(4/4) = 1
-        assert_eq!(estimate_tokens("abcde"), 2);  // 5 → ceil(5/4) = 2
-        assert_eq!(estimate_tokens(&"x".repeat(400)), 100);
-    }
 
     #[test]
     fn context_window_known_and_fallback() {
@@ -179,7 +173,7 @@ mod tests {
         ).unwrap();
         assert!(with_sys.estimated > without_sys.estimated,
             "system prompt should raise estimate");
-        assert_eq!(with_sys.estimated - without_sys.estimated, estimate_tokens(&sys));
+        assert_eq!(with_sys.estimated - without_sys.estimated, shared::model_matrix::ModelMatrix::estimate_tokens(&sys));
     }
 
     #[test]

crates/aibridge/src/routing.rs

@@ -70,15 +70,19 @@ pub struct RouteDecision {
 }
 
 fn glob_match(pattern: &str, name: &str) -> bool {
-    if pattern.contains('*') {
+    if !pattern.contains('*') { return pattern == name; }
     let parts: Vec<&str> = pattern.split('*').collect();
-        if parts.len() == 2 {
+    // Multi-* support: first must be prefix, last must be suffix, each
-            return name.starts_with(parts[0]) && name.ends_with(parts[1]);
+    // interior piece must appear in order. Fixes the iter-9 finding
-        } else if parts.len() == 1 {
+    // where gpt-*-large* silently fell through to an exact-match path.
-            return name.starts_with(parts[0]) || name.ends_with(parts[1]);
+    // Also removes the dead `parts.len() == 1` branch that accessed
-        }
+    // parts[1] and would panic if ever reached (unreachable today
-    }
+    // since split('*') on a string containing '*' always yields ≥2).
-    pattern == name
+    if !name.starts_with(parts[0]) || !name.ends_with(parts.last().unwrap()) { return false; }
+    let mut cursor = parts[0].len();
+    parts[1..parts.len() - 1].iter().all(|mid| {
+        name[cursor..].find(mid).map(|pos| { cursor += pos + mid.len(); true }).unwrap_or(false)
+    })
 }
 
 impl Default for RoutingRule {
@@ -91,3 +95,25 @@ impl Default for RoutingRule {
         }
     }
 }
+
+#[cfg(test)]
+mod glob_match_tests {
+    use super::glob_match;
+
+    #[test] fn exact_match() { assert!(glob_match("gpt-oss:120b", "gpt-oss:120b")); }
+    #[test] fn exact_mismatch() { assert!(!glob_match("a", "b")); }
+    #[test] fn leading_wildcard() { assert!(glob_match("*:120b", "gpt-oss:120b")); }
+    #[test] fn trailing_wildcard() { assert!(glob_match("gpt-oss:*", "gpt-oss:120b")); }
+    #[test] fn bare_wildcard() { assert!(glob_match("*", "anything")); }
+    #[test] fn multi_wildcard_in_order() { assert!(glob_match("gpt-*-oss-*", "gpt-4-oss-120b")); }
+    #[test] fn multi_wildcard_wrong_order() { assert!(!glob_match("b*a*", "abba")); }
+    #[test] fn multi_wildcard_panic_safety() {
+        // Regression: earlier impl had an unreachable `parts.len() == 1`
+        // branch that indexed parts[1] — would panic if ever hit. Now
+        // the split('*') invariant guarantees ≥2 parts when * present,
+        // and we handle all N-part cases explicitly.
+        assert!(glob_match("a*b*c", "abc"));
+        assert!(glob_match("a*b*c", "axxxbxxxc"));
+        assert!(!glob_match("a*b*c", "xxxbxxx"));
+    }
+}

crates/aibridge/src/tree_split.rs

@@ -19,8 +19,9 @@
 //! we bubble the error up rather than silently truncating. That's the
 //! whole point of Phase 21.
 
-use crate::context::{assert_context_budget, BudgetOpts, estimate_tokens, overflow_message,
+use crate::context::{assert_context_budget, BudgetOpts, overflow_message,
                      DEFAULT_MAX_TOKENS, DEFAULT_SAFETY_MARGIN};
+use shared::model_matrix::ModelMatrix;
 use crate::continuation::{generate_continuable, ContinuableOpts, ResponseShape, TextGenerator};
 
 /// Callback signatures — caller supplies closures that stitch the
@@ -80,12 +81,12 @@ pub struct TreeSplitResult {
 /// by `\n— shard N/M digest —\n` so we can find the first one and
 /// chop everything before its successor.
 fn truncate_scratchpad(scratchpad: &mut String, budget_tokens: usize) -> bool {
-    if estimate_tokens(scratchpad) <= budget_tokens { return false; }
+    if ModelMatrix::estimate_tokens(scratchpad) <= budget_tokens { return false; }
     // Find the second delimiter — everything before it gets dropped.
     const DELIM_PREFIX: &str = "\n— shard ";
     let mut cursor = 0;
     let mut truncated = false;
-    while estimate_tokens(&scratchpad[cursor..]) > budget_tokens {
+    while ModelMatrix::estimate_tokens(&scratchpad[cursor..]) > budget_tokens {
         // Skip past a leading delimiter (if we're sitting on one from
         // a previous iteration), then find the next.
         let search_from = cursor + if scratchpad[cursor..].starts_with(DELIM_PREFIX) {
@@ -278,7 +279,7 @@ mod tests {
         // Scratchpad should still fit roughly within the budget
         // (post-truncation); the estimator uses chars/4 so the bound
         // is ~budget*4 chars. Give some slack for the delimiter.
-        let scratchpad_tokens = estimate_tokens(&result.scratchpad);
+        let scratchpad_tokens = ModelMatrix::estimate_tokens(&result.scratchpad);
         assert!(scratchpad_tokens <= opts.scratchpad_budget * 2,
             "scratchpad {} tokens vs budget {}", scratchpad_tokens, opts.scratchpad_budget);
     }

crates/gateway/src/access.rs

@@ -93,8 +93,7 @@ impl AccessControl {
         self.roles.write().await.insert(role.agent_name.clone(), role);
     }
 
-    /// Get an agent's role.
+    /// Get an agent's role. Called by `GET /access/roles/{agent}`.
-    #[allow(dead_code)]
     pub async fn get_role(&self, agent: &str) -> Option<AgentRole> {
         self.roles.read().await.get(agent).cloned()
     }
@@ -152,7 +151,9 @@ impl AccessControl {
         log[start..].iter().rev().cloned().collect()
     }
 
-    #[allow(dead_code)]
+    /// Reports whether access-control enforcement is active.
+    /// Called by `GET /access/enabled` — ops tooling / dashboards poll
+    /// this to confirm the auth posture of the running gateway.
     pub fn is_enabled(&self) -> bool {
         self.enabled
     }

crates/gateway/src/access_service.rs

@@ -1,6 +1,6 @@
 use axum::{
     Json, Router,
-    extract::{Query, State},
+    extract::{Path, Query, State},
     http::StatusCode,
     response::IntoResponse,
     routing::{get, post},
@@ -13,6 +13,12 @@ pub fn router(ac: AccessControl) -> Router {
     Router::new()
         .route("/roles", get(list_roles))
         .route("/roles", post(set_role))
+        // Scrum iter 11 / P13-001 finding: get_role was #[allow(dead_code)]
+        // because nothing called it — dead until exposed. Route activates it.
+        // Returns 404 when the agent isn't registered so clients can
+        // distinguish "missing role" from "access denied."
+        .route("/roles/{agent}", get(get_role))
+        .route("/enabled", get(enabled_status))
         .route("/audit", get(query_audit))
         .route("/check", post(check_access))
         .with_state(ac)
@@ -60,3 +66,17 @@ async fn check_access(
         "allowed": allowed,
     }))
 }
+
+async fn get_role(
+    State(ac): State<AccessControl>,
+    Path(agent): Path<String>,
+) -> impl IntoResponse {
+    match ac.get_role(&agent).await {
+        Some(role) => Ok(Json(role)),
+        None => Err((StatusCode::NOT_FOUND, format!("no role registered for agent '{agent}'"))),
+    }
+}
+
+async fn enabled_status(State(ac): State<AccessControl>) -> impl IntoResponse {
+    Json(serde_json::json!({ "enabled": ac.is_enabled() }))
+}

crates/gateway/src/execution_loop/mod.rs

@@ -82,7 +82,7 @@ pub struct Fill {
 pub enum RespondOutcome {
     Ok { artifact: serde_json::Value, log: Vec<LogEntry> },
     Failed { reason: String, log: Vec<LogEntry> },
-    #[allow(dead_code)] // reserved for Phase 42 truth-gate (step 6)
+    // Constructed by the truth-gate check in run_inner (step 6, 2026-04-24).
     Blocked { reason: String, log: Vec<LogEntry> },
 }
 
@@ -153,7 +153,21 @@ impl ExecutionLoop {
             .as_deref().unwrap_or(DEFAULT_REVIEWER_MODEL).to_string();
         let max_turns = self.req.max_turns.unwrap_or(DEFAULT_MAX_TURNS);
 
-        // --- (6) TRUTH GATE — PORT FROM Phase 42 (TODO) ---
+        // --- (6) TRUTH GATE — Phase 42 wiring (2026-04-24) ---
+        // Evaluate truth rules for the request's task_class against a
+        // ctx built from the spec. Any rule whose condition holds AND
+        // whose action is Reject/Block short-circuits to Blocked before
+        // the executor loop runs. Mirrors queryd/service.rs SQL gate.
+        let truth_store = truth::default_truth_store();
+        for outcome in truth_store.evaluate(&self.req.task_class, &self.req.spec) {
+            if !outcome.passed { continue; }
+            if let truth::RuleAction::Reject { message } | truth::RuleAction::Block { message } = &outcome.action {
+                let reason = format!("truth rule {} blocked: {message}", outcome.rule_id);
+                self.append(LogEntry::new(0, "system", "truth", "block",
+                    serde_json::json!({ "rule_id": outcome.rule_id, "reason": reason.clone() })));
+                return Ok(RespondOutcome::Blocked { reason, log: self.log.clone() });
+            }
+        }
 
         // --- (1) PLAYBOOK BOOST ---
         let boost = self.fetch_playbook_boost(&self.req.operation).await.unwrap_or_default();
@@ -1533,8 +1547,15 @@ mod tests {
         ];
         let p = build_executor_prompt(&req, &[], &log);
         assert!(p.contains("CANDIDATES SURFACED SO FAR"));
-        assert!(p.contains("W-1 Alice Smith"));
+        // Prompt format deliberately separates name from doc_id now —
-        assert!(p.contains("Toledo, OH"));
+        // the line reads `name="Alice Smith" ... (vector doc_id=W-1)`
+        // so the executor prompt explicitly tells the model NOT to
+        // conflate doc_id with workers_500k.worker_id. Assertion was
+        // expecting the old concatenated format; update to match the
+        // semantic contract (both tokens present, any order).
+        assert!(p.contains("Alice Smith"));
+        assert!(p.contains("W-1"));
+        assert!(p.contains("Toledo"));
     }
 
     #[test]

crates/gateway/src/main.rs

@@ -68,6 +68,27 @@ async fn main() {
     let access = access::AccessControl::new(config.auth.enabled);
     access.register_defaults().await;
 
+    // Phase 42 — file-backed truth rules. Probes the `truth/` directory
+    // at repo root (or $LAKEHOUSE_TRUTH_DIR override) and logs how many
+    // rules load. Current request paths still build their own stores
+    // via truth::default_truth_store() / truth::sql_query_guard_store();
+    // the composed-at-boot store gets plumbed through V1State in a
+    // follow-up. This boot probe catches parse errors + duplicate-ID
+    // collisions early rather than at first request.
+    {
+        let truth_dir = std::env::var("LAKEHOUSE_TRUTH_DIR")
+            .unwrap_or_else(|_| "/home/profit/lakehouse/truth".to_string());
+        if std::path::Path::new(&truth_dir).exists() {
+            let mut probe_store = truth::default_truth_store();
+            match truth::loader::load_from_dir(&mut probe_store, &truth_dir) {
+                Ok(n) => tracing::info!("truth: loaded {n} file-backed rule(s) from {truth_dir}"),
+                Err(e) => tracing::warn!("truth: failed to load rules from {truth_dir}: {e}"),
+            }
+        } else {
+            tracing::debug!("truth: no rule dir at {truth_dir}, skipping file-backed load");
+        }
+    }
+
     // Workspace manager for agent-specific overlays
     let workspace_mgr = queryd::workspace::WorkspaceManager::new(store.clone());
     if let Err(e) = workspace_mgr.rebuild().await {
@@ -93,6 +114,12 @@ async fn main() {
     // operators call POST /vectors/playbook_memory/rebuild to populate.
     let pbm = vectord::playbook_memory::PlaybookMemory::new(store.clone());
     let _ = pbm.load_from_storage().await;
+    // Pathway memory — consensus-designed sidecar for full-context
+    // backtracking + hot-swap of successful review pathways. Same
+    // load-on-boot pattern as playbook_memory: empty state is fine,
+    // operators start populating via scrum_master_pipeline.ts.
+    let pwm = vectord::pathway_memory::PathwayMemory::new(store.clone());
+    let _ = pwm.load_from_storage().await;
 
     // Phase 16.2: spawn the autotune agent. When config.agent.enabled=false
     // this returns a handle that drops triggers silently — no surprise load.
@@ -178,17 +205,19 @@ async fn main() {
                 bucket_registry.clone(), index_reg.clone(),
             ),
             playbook_memory: pbm,
+            pathway_memory: pwm,
             embed_semaphore: std::sync::Arc::new(tokio::sync::Semaphore::new(1)),
         }))
         .nest("/workspaces", queryd::workspace_service::router(workspace_mgr))
         .nest("/journal", journald::service::router(journal))
         .nest("/access", access_service::router(access))
         .nest("/tools", tools::service::router({
-            let tool_reg = tools::registry::ToolRegistry::new_with_defaults();
+            let tool_reg = tools::registry::ToolRegistry::new();
             tool_reg.register_defaults().await;
             tools::ToolState {
                 registry: tool_reg,
                 query_fn: tools::QueryExecutor::new(engine.clone()),
+                truth: std::sync::Arc::new(truth::sql_query_guard_store()),
             }
         }))
         // Phase 38 — Universal API skeleton. Thin OpenAI-compatible
@@ -210,6 +239,38 @@ async fn main() {
                 }
                 k
             },
+            openrouter_key: {
+                // 2026-04-24 free-tier rescue rung for iter 5+. Shares
+                // the LLM Team UI's OPENROUTER_API_KEY so both systems
+                // draw from one quota.
+                let k = v1::openrouter::resolve_openrouter_key();
+                if k.is_some() {
+                    tracing::info!("v1: OpenRouter key loaded — /v1/chat provider=openrouter enabled");
+                } else {
+                    tracing::warn!("v1: no OpenRouter key — openrouter rescue rung will 503");
+                }
+                k
+            },
+            gemini_key: {
+                // Phase 40 provider. GEMINI_API_KEY in env or .env.
+                let k = v1::gemini::resolve_gemini_key();
+                if k.is_some() {
+                    tracing::info!("v1: Gemini key loaded — /v1/chat provider=gemini enabled");
+                } else {
+                    tracing::debug!("v1: no Gemini key — provider=gemini will 503");
+                }
+                k
+            },
+            claude_key: {
+                // Phase 40 provider. ANTHROPIC_API_KEY in env or .env.
+                let k = v1::claude::resolve_claude_key();
+                if k.is_some() {
+                    tracing::info!("v1: Claude key loaded — /v1/chat provider=claude enabled");
+                } else {
+                    tracing::debug!("v1: no Claude key — provider=claude will 503");
+                }
+                k
+            },
             // Phase 40 early deliverable — Langfuse trace emitter.
             // Defaults match mcp-server/tracing.ts conventions so
             // gateway traces land in the same staffing project.

crates/gateway/src/tools/mod.rs

@@ -3,12 +3,18 @@ pub mod service;
 
 use queryd::context::QueryEngine;
 use arrow::json::writer::{JsonArray, Writer as JsonWriter};
+use std::sync::Arc;
+use truth::TruthStore;
 
 /// State for the tool system.
 #[derive(Clone)]
 pub struct ToolState {
     pub registry: registry::ToolRegistry,
     pub query_fn: QueryExecutor,
+    /// SQL guard (shared with queryd). Mirrors the queryd /sql truth
+    /// gate from P42-002 (9cc0ceb) — tools also execute model-
+    /// originated SQL, need the same destructive-verb block.
+    pub truth: Arc<TruthStore>,
 }
 
 /// Wraps QueryEngine to provide a simple execute interface for tools.

crates/gateway/src/tools/registry.rs

@@ -67,25 +67,14 @@ pub struct ToolRegistry {
 }
 
 impl ToolRegistry {
-    #[allow(dead_code)]
+    /// Build an empty registry. Callers in an async context should follow
+    /// this with `.register_defaults().await` if they want the built-in
+    /// staffing tools pre-installed — main.rs does exactly that.
     pub fn new() -> Self {
-        let registry = Self {
+        Self {
             tools: Arc::new(RwLock::new(HashMap::new())),
             audit_log: Arc::new(RwLock::new(Vec::new())),
-        };
-        // Register built-in staffing tools
-        tokio::task::block_in_place(|| {
-            tokio::runtime::Handle::current().block_on(registry.register_defaults())
-        });
-        registry
         }
-
-    pub fn new_with_defaults() -> Self {
-        let registry = Self {
-            tools: Arc::new(RwLock::new(HashMap::new())),
-            audit_log: Arc::new(RwLock::new(Vec::new())),
-        };
-        registry
     }
 
     /// Register default staffing tools.

crates/gateway/src/tools/service.rs

@@ -92,6 +92,32 @@ async fn call_tool(
         }
     };
 
+    // Truth gate — same contract as queryd /sql (P42-002). Rejects
+    // destructive verbs + empty SQL. Scrum iter 11 CF-1 + CF-2 on this
+    // file: tools executed model-provided SQL parameters without any
+    // validation. Close the gap here so the parallel surface has the
+    // same safety floor as queryd.
+    let ctx = serde_json::json!({ "sql": sql });
+    for outcome in state.truth.evaluate("sql_query", &ctx) {
+        if outcome.passed {
+            if let truth::RuleAction::Reject { message } | truth::RuleAction::Block { message } = &outcome.action {
+                tracing::warn!("tool {name}: SQL blocked by truth gate ({}): {message}", outcome.rule_id);
+                state.registry.log_invocation(ToolInvocation {
+                    id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
+                    tool_name: name.clone(),
+                    agent: req.agent.clone(),
+                    params: req.params.clone(),
+                    permission: tool.permission.clone(),
+                    timestamp: chrono::Utc::now(),
+                    success: false,
+                    error: Some(format!("truth gate: {message}")),
+                    rows_returned: None,
+                }).await;
+                return Err((StatusCode::FORBIDDEN, message.clone()));
+            }
+        }
+    }
+
     // Execute via query engine
     let result = state.query_fn.execute(&sql).await;
 

crates/gateway/src/v1/claude.rs

@@ -0,0 +1,222 @@
+//! Claude (Anthropic) adapter.
+//!
+//! POST `https://api.anthropic.com/v1/messages`. Auth via `x-api-key`
+//! header (not bearer) + required `anthropic-version` header. Payload
+//! is NOT OpenAI-compatible — response text lives at
+//! `content[0].text`. Phase 40 deliverable. System prompts travel in
+//! a top-level `system` field, separate from the `messages` array.
+
+use std::time::Duration;
+use serde::{Deserialize, Serialize};
+
+use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
+
+const CLAUDE_BASE_URL: &str = "https://api.anthropic.com/v1";
+const CLAUDE_API_VERSION: &str = "2023-06-01";
+const CLAUDE_TIMEOUT_SECS: u64 = 180;
+
+pub fn resolve_claude_key() -> Option<String> {
+    if let Ok(k) = std::env::var("ANTHROPIC_API_KEY") {
+        if !k.trim().is_empty() { return Some(k.trim().to_string()); }
+    }
+    for path in ["/home/profit/.env", "/root/.env"] {
+        if let Ok(raw) = std::fs::read_to_string(path) {
+            for line in raw.lines() {
+                if let Some(rest) = line.strip_prefix("ANTHROPIC_API_KEY=") {
+                    let k = rest.trim().trim_matches('"').trim_matches('\'');
+                    if !k.is_empty() { return Some(k.to_string()); }
+                }
+            }
+        }
+    }
+    None
+}
+
+pub async fn chat(
+    key: &str,
+    req: &ChatRequest,
+) -> Result<ChatResponse, String> {
+    // Strip the "claude/" prefix if the caller used the namespaced form.
+    let model = req.model.strip_prefix("claude/").unwrap_or(&req.model).to_string();
+
+    // Anthropic carries system prompts outside the messages array.
+    // Concatenate any system-role messages into a single system string;
+    // keep user + assistant messages in `messages`.
+    let mut system_parts: Vec<String> = Vec::new();
+    let mut msgs: Vec<AnMessage> = Vec::new();
+    for m in &req.messages {
+        if m.role == "system" {
+            system_parts.push(m.content.clone());
+        } else {
+            // Anthropic expects strictly "user" or "assistant"; anything
+            // else we normalize to "user".
+            let role = if m.role == "assistant" { "assistant" } else { "user" };
+            msgs.push(AnMessage { role: role.to_string(), content: m.content.clone() });
+        }
+    }
+    let system = if system_parts.is_empty() {
+        None
+    } else {
+        Some(system_parts.join("\n\n"))
+    };
+
+    let body = AnChatBody {
+        model: model.clone(),
+        messages: msgs,
+        max_tokens: req.max_tokens.unwrap_or(800),
+        temperature: req.temperature.unwrap_or(0.3),
+        system,
+    };
+
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(CLAUDE_TIMEOUT_SECS))
+        .build()
+        .map_err(|e| format!("build client: {e}"))?;
+
+    let t0 = std::time::Instant::now();
+    let resp = client
+        .post(format!("{}/messages", CLAUDE_BASE_URL))
+        .header("x-api-key", key)
+        .header("anthropic-version", CLAUDE_API_VERSION)
+        .json(&body)
+        .send()
+        .await
+        .map_err(|e| format!("api.anthropic.com unreachable: {e}"))?;
+
+    let status = resp.status();
+    if !status.is_success() {
+        let body = resp.text().await.unwrap_or_else(|_| "?".into());
+        return Err(format!("claude {}: {}", status, body));
+    }
+
+    let parsed: AnChatResponse = resp.json().await
+        .map_err(|e| format!("invalid claude response: {e}"))?;
+
+    let latency_ms = t0.elapsed().as_millis();
+    let text = parsed.content.into_iter()
+        .find(|b| b.block_type == "text")
+        .map(|b| b.text)
+        .unwrap_or_default();
+
+    let prompt_tokens = parsed.usage.as_ref().map(|u| u.input_tokens).unwrap_or_else(|| {
+        let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
+        ((chars + 3) / 4) as u32
+    });
+    let completion_tokens = parsed.usage.as_ref().map(|u| u.output_tokens).unwrap_or_else(|| {
+        ((text.chars().count() + 3) / 4) as u32
+    });
+
+    tracing::info!(
+        target: "v1.chat",
+        provider = "claude",
+        model = %model,
+        prompt_tokens,
+        completion_tokens,
+        latency_ms = latency_ms as u64,
+        "claude chat completed",
+    );
+
+    Ok(ChatResponse {
+        id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
+        object: "chat.completion",
+        created: chrono::Utc::now().timestamp(),
+        model,
+        choices: vec![Choice {
+            index: 0,
+            message: Message { role: "assistant".into(), content: text },
+            finish_reason: parsed.stop_reason.unwrap_or_else(|| "stop".into()),
+        }],
+        usage: UsageBlock {
+            prompt_tokens,
+            completion_tokens,
+            total_tokens: prompt_tokens + completion_tokens,
+        },
+    })
+}
+
+// -- Anthropic Messages API wire shapes --
+
+#[derive(Serialize)]
+struct AnChatBody {
+    model: String,
+    messages: Vec<AnMessage>,
+    max_tokens: u32,
+    temperature: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    system: Option<String>,
+}
+
+#[derive(Serialize)]
+struct AnMessage { role: String, content: String }
+
+#[derive(Deserialize)]
+struct AnChatResponse {
+    content: Vec<AnContentBlock>,
+    #[serde(default, rename = "stop_reason")]
+    stop_reason: Option<String>,
+    #[serde(default)]
+    usage: Option<AnUsage>,
+}
+
+#[derive(Deserialize)]
+struct AnContentBlock {
+    #[serde(rename = "type")]
+    block_type: String,
+    #[serde(default)]
+    text: String,
+}
+
+#[derive(Deserialize)]
+struct AnUsage { input_tokens: u32, output_tokens: u32 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resolve_claude_key_does_not_panic() {
+        let _ = resolve_claude_key();
+    }
+
+    #[test]
+    fn chat_body_serializes_with_separate_system() {
+        let body = AnChatBody {
+            model: "claude-3-5-sonnet-latest".into(),
+            messages: vec![
+                AnMessage { role: "user".into(), content: "hi".into() },
+            ],
+            max_tokens: 800,
+            temperature: 0.3,
+            system: Some("You are helpful.".into()),
+        };
+        let json = serde_json::to_string(&body).unwrap();
+        assert!(json.contains("\"system\":\"You are helpful.\""));
+        assert!(json.contains("\"messages\""));
+        assert!(json.contains("\"max_tokens\":800"));
+    }
+
+    #[test]
+    fn body_omits_system_when_none() {
+        let body = AnChatBody {
+            model: "claude-3-5-sonnet-latest".into(),
+            messages: vec![AnMessage { role: "user".into(), content: "hi".into() }],
+            max_tokens: 800,
+            temperature: 0.3,
+            system: None,
+        };
+        let json = serde_json::to_string(&body).unwrap();
+        assert!(!json.contains("\"system\""), "system field should be skipped when None: {json}");
+    }
+
+    #[test]
+    fn model_prefix_strip_preserves_bare_names() {
+        let cases = [
+            ("claude/claude-3-5-sonnet-latest", "claude-3-5-sonnet-latest"),
+            ("claude-3-5-sonnet-latest", "claude-3-5-sonnet-latest"),
+        ];
+        for (input, expected) in cases {
+            let out = input.strip_prefix("claude/").unwrap_or(input);
+            assert_eq!(out, expected);
+        }
+    }
+}

crates/gateway/src/v1/gemini.rs

@@ -0,0 +1,230 @@
+//! Gemini adapter — Google's Generative Language API.
+//!
+//! POST `https://generativelanguage.googleapis.com/v1beta/models/
+//! {model}:generateContent?key=<API_KEY>`. Auth via query-string key
+//! (not bearer). Payload shape is NOT OpenAI-compatible — we map
+//! messages → contents + parts, extract response from `candidates[0]
+//! .content.parts[0].text`. Phase 40 deliverable; gate: `/v1/chat`
+//! with a prefixed or explicit gemini model returns normally.
+
+use std::time::Duration;
+use serde::{Deserialize, Serialize};
+
+use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
+
+const GEMINI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta";
+const GEMINI_TIMEOUT_SECS: u64 = 180;
+
+pub fn resolve_gemini_key() -> Option<String> {
+    if let Ok(k) = std::env::var("GEMINI_API_KEY") {
+        if !k.trim().is_empty() { return Some(k.trim().to_string()); }
+    }
+    for path in ["/home/profit/.env", "/root/.env"] {
+        if let Ok(raw) = std::fs::read_to_string(path) {
+            for line in raw.lines() {
+                if let Some(rest) = line.strip_prefix("GEMINI_API_KEY=") {
+                    let k = rest.trim().trim_matches('"').trim_matches('\'');
+                    if !k.is_empty() { return Some(k.to_string()); }
+                }
+            }
+        }
+    }
+    None
+}
+
+pub async fn chat(
+    key: &str,
+    req: &ChatRequest,
+) -> Result<ChatResponse, String> {
+    // Strip the "gemini/" prefix if the caller used the namespaced form.
+    let model = req.model.strip_prefix("gemini/").unwrap_or(&req.model).to_string();
+
+    // Gemini splits system prompt from conversation differently.
+    // Simplest working mapping: concatenate any system messages at the
+    // top of a single user turn, then append user/assistant turns as
+    // separate contents entries. Covers the common single-turn case
+    // the scrum pipeline uses.
+    let mut contents: Vec<GmContent> = Vec::new();
+    for m in &req.messages {
+        let role = match m.role.as_str() {
+            "system" | "user" => "user",
+            _ => "model",
+        };
+        contents.push(GmContent {
+            role: role.to_string(),
+            parts: vec![GmPart { text: m.content.clone() }],
+        });
+    }
+
+    let body = GmChatBody {
+        contents,
+        generation_config: GmGenerationConfig {
+            temperature: req.temperature.unwrap_or(0.3),
+            max_output_tokens: req.max_tokens.unwrap_or(800),
+        },
+    };
+
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(GEMINI_TIMEOUT_SECS))
+        .build()
+        .map_err(|e| format!("build client: {e}"))?;
+
+    let url = format!("{}/models/{}:generateContent?key={}", GEMINI_BASE_URL, model, key);
+    let t0 = std::time::Instant::now();
+    let resp = client
+        .post(&url)
+        .json(&body)
+        .send()
+        .await
+        .map_err(|e| format!("generativelanguage.googleapis.com unreachable: {e}"))?;
+
+    let status = resp.status();
+    if !status.is_success() {
+        let body = resp.text().await.unwrap_or_else(|_| "?".into());
+        return Err(format!("gemini {}: {}", status, body));
+    }
+
+    let parsed: GmChatResponse = resp.json().await
+        .map_err(|e| format!("invalid gemini response: {e}"))?;
+
+    let latency_ms = t0.elapsed().as_millis();
+    let candidate = parsed.candidates.into_iter().next()
+        .ok_or_else(|| "gemini returned no candidates".to_string())?;
+    let text = candidate.content.parts.into_iter()
+        .next()
+        .map(|p| p.text)
+        .unwrap_or_default();
+
+    let prompt_tokens = parsed.usage_metadata.as_ref()
+        .map(|u| u.prompt_token_count)
+        .unwrap_or_else(|| {
+            let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
+            ((chars + 3) / 4) as u32
+        });
+    let completion_tokens = parsed.usage_metadata.as_ref()
+        .map(|u| u.candidates_token_count)
+        .unwrap_or_else(|| ((text.chars().count() + 3) / 4) as u32);
+
+    tracing::info!(
+        target: "v1.chat",
+        provider = "gemini",
+        model = %model,
+        prompt_tokens,
+        completion_tokens,
+        latency_ms = latency_ms as u64,
+        "gemini chat completed",
+    );
+
+    Ok(ChatResponse {
+        id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
+        object: "chat.completion",
+        created: chrono::Utc::now().timestamp(),
+        model,
+        choices: vec![Choice {
+            index: 0,
+            message: Message { role: "assistant".into(), content: text },
+            finish_reason: candidate.finish_reason.unwrap_or_else(|| "stop".into()),
+        }],
+        usage: UsageBlock {
+            prompt_tokens,
+            completion_tokens,
+            total_tokens: prompt_tokens + completion_tokens,
+        },
+    })
+}
+
+// -- Gemini wire shapes --
+
+#[derive(Serialize)]
+struct GmChatBody {
+    contents: Vec<GmContent>,
+    #[serde(rename = "generationConfig")]
+    generation_config: GmGenerationConfig,
+}
+
+#[derive(Serialize)]
+struct GmContent {
+    role: String,
+    parts: Vec<GmPart>,
+}
+
+#[derive(Serialize)]
+struct GmPart { text: String }
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct GmGenerationConfig {
+    temperature: f64,
+    max_output_tokens: u32,
+}
+
+#[derive(Deserialize)]
+struct GmChatResponse {
+    candidates: Vec<GmCandidate>,
+    #[serde(default, rename = "usageMetadata")]
+    usage_metadata: Option<GmUsage>,
+}
+
+#[derive(Deserialize)]
+struct GmCandidate {
+    content: GmContentResp,
+    #[serde(default, rename = "finishReason")]
+    finish_reason: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct GmContentResp { parts: Vec<GmPartResp> }
+
+#[derive(Deserialize)]
+struct GmPartResp { #[serde(default)] text: String }
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct GmUsage {
+    prompt_token_count: u32,
+    candidates_token_count: u32,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resolve_gemini_key_does_not_panic() {
+        let _ = resolve_gemini_key();
+    }
+
+    #[test]
+    fn chat_body_serializes_to_gemini_shape() {
+        let body = GmChatBody {
+            contents: vec![
+                GmContent {
+                    role: "user".into(),
+                    parts: vec![GmPart { text: "hello".into() }],
+                },
+            ],
+            generation_config: GmGenerationConfig {
+                temperature: 0.3,
+                max_output_tokens: 800,
+            },
+        };
+        let json = serde_json::to_string(&body).unwrap();
+        assert!(json.contains("\"contents\""));
+        assert!(json.contains("\"parts\""));
+        // camelCase per Gemini API
+        assert!(json.contains("\"generationConfig\""));
+        assert!(json.contains("\"maxOutputTokens\":800"));
+    }
+
+    #[test]
+    fn model_prefix_strip_preserves_bare_names() {
+        let cases = [
+            ("gemini/gemini-2.0-flash", "gemini-2.0-flash"),
+            ("gemini-2.0-flash", "gemini-2.0-flash"),
+        ];
+        for (input, expected) in cases {
+            let out = input.strip_prefix("gemini/").unwrap_or(input);
+            assert_eq!(out, expected);
+        }
+    }
+}

crates/gateway/src/v1/mod.rs

@@ -13,6 +13,9 @@
 
 pub mod ollama;
 pub mod ollama_cloud;
+pub mod openrouter;
+pub mod gemini;
+pub mod claude;
 pub mod langfuse_trace;
 pub mod respond;
 pub mod truth;
@@ -36,6 +39,19 @@ pub struct V1State {
     /// Ollama Cloud bearer token. Loaded at startup via
     /// `ollama_cloud::resolve_cloud_key()`. None = cloud routes 503.
     pub ollama_cloud_key: Option<String>,
+    /// OpenRouter bearer token — free-tier rescue rung. Loaded at
+    /// startup via `openrouter::resolve_openrouter_key()`. None means
+    /// provider="openrouter" calls 503 rather than attempt. Same key
+    /// sourcing as LLM Team UI so the two share one API quota.
+    pub openrouter_key: Option<String>,
+    /// Gemini API key (Google Generative Language). Loaded at startup
+    /// via `gemini::resolve_gemini_key()`. None = provider="gemini"
+    /// calls 503. Phase 40 deliverable.
+    pub gemini_key: Option<String>,
+    /// Anthropic Claude API key. Loaded at startup via
+    /// `claude::resolve_claude_key()`. None = provider="claude" calls
+    /// 503. Phase 40 deliverable.
+    pub claude_key: Option<String>,
     /// Phase 40 early deliverable — Langfuse client. None = tracing
     /// disabled (keys missing or container unreachable). Traces are
     /// fire-and-forget: never block the response path.
@@ -78,7 +94,7 @@ pub struct Message {
     pub content: String,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct ChatRequest {
     pub model: String,
     pub messages: Vec<Message>,
@@ -134,6 +150,96 @@ pub struct UsageBlock {
 
 // -- Handlers --
 
+/// Phase 39: resolve (provider, effective_model) from a ChatRequest.
+///
+/// Explicit `req.provider` wins. If absent, infer from a model-name
+/// prefix: "openrouter/..." → openrouter (strip prefix), "cloud/..." →
+/// ollama_cloud (strip prefix). Bare names default to "ollama".
+///
+/// The stripped model is what the upstream adapter expects:
+/// OpenRouter's API wants "openai/gpt-4o-mini", not
+/// "openrouter/openai/gpt-4o-mini".
+fn resolve_provider(req: &ChatRequest) -> (String, String) {
+    if let Some(p) = req.provider.as_deref() {
+        return (p.to_ascii_lowercase(), req.model.clone());
+    }
+    if let Some(rest) = req.model.strip_prefix("openrouter/") {
+        return ("openrouter".to_string(), rest.to_string());
+    }
+    if let Some(rest) = req.model.strip_prefix("cloud/") {
+        return ("ollama_cloud".to_string(), rest.to_string());
+    }
+    if let Some(rest) = req.model.strip_prefix("gemini/") {
+        return ("gemini".to_string(), rest.to_string());
+    }
+    if let Some(rest) = req.model.strip_prefix("claude/") {
+        return ("claude".to_string(), rest.to_string());
+    }
+    ("ollama".to_string(), req.model.clone())
+}
+
+#[cfg(test)]
+mod resolve_provider_tests {
+    use super::*;
+
+    fn mk_req(provider: Option<&str>, model: &str) -> ChatRequest {
+        ChatRequest {
+            model: model.to_string(),
+            messages: vec![],
+            temperature: None,
+            max_tokens: None,
+            stream: None,
+            think: None,
+            provider: provider.map(|s| s.to_string()),
+        }
+    }
+
+    #[test]
+    fn explicit_provider_wins() {
+        let r = mk_req(Some("openrouter"), "qwen3.5:latest");
+        assert_eq!(resolve_provider(&r), ("openrouter".into(), "qwen3.5:latest".into()));
+    }
+
+    #[test]
+    fn bare_model_defaults_to_ollama() {
+        let r = mk_req(None, "qwen3.5:latest");
+        assert_eq!(resolve_provider(&r), ("ollama".into(), "qwen3.5:latest".into()));
+    }
+
+    #[test]
+    fn openrouter_prefix_infers_and_strips() {
+        let r = mk_req(None, "openrouter/openai/gpt-4o-mini");
+        assert_eq!(resolve_provider(&r), ("openrouter".into(), "openai/gpt-4o-mini".into()));
+    }
+
+    #[test]
+    fn cloud_prefix_infers_and_strips() {
+        let r = mk_req(None, "cloud/kimi-k2:1t");
+        assert_eq!(resolve_provider(&r), ("ollama_cloud".into(), "kimi-k2:1t".into()));
+    }
+
+    #[test]
+    fn explicit_provider_preserves_full_model_even_with_prefix() {
+        // If caller provides both provider and a model with a prefix,
+        // trust them — don't strip. The adapter will get the full model
+        // string as-is.
+        let r = mk_req(Some("openrouter"), "openrouter/openai/gpt-4o-mini");
+        assert_eq!(resolve_provider(&r), ("openrouter".into(), "openrouter/openai/gpt-4o-mini".into()));
+    }
+
+    #[test]
+    fn gemini_prefix_infers_and_strips() {
+        let r = mk_req(None, "gemini/gemini-2.0-flash");
+        assert_eq!(resolve_provider(&r), ("gemini".into(), "gemini-2.0-flash".into()));
+    }
+
+    #[test]
+    fn claude_prefix_infers_and_strips() {
+        let r = mk_req(None, "claude/claude-3-5-sonnet-latest");
+        assert_eq!(resolve_provider(&r), ("claude".into(), "claude-3-5-sonnet-latest".into()));
+    }
+}
+
 async fn chat(
     State(state): State<V1State>,
     Json(req): Json<ChatRequest>,
@@ -145,13 +251,29 @@ async fn chat(
         tracing::warn!("/v1/chat: stream=true requested but Phase 38 returns non-streaming");
     }
 
-    let provider = req.provider.as_deref().unwrap_or("ollama").to_ascii_lowercase();
+    // Provider resolution: explicit `req.provider` wins; otherwise
+    // infer from a model-name prefix. Phase 39 PRD gate example:
+    // `model: "openrouter/openai/gpt-4o-mini"` → provider "openrouter",
+    // adapter gets the stripped "openai/gpt-4o-mini".
+    let (provider, effective_model) = resolve_provider(&req);
     let start_time = chrono::Utc::now();
     let start_instant = std::time::Instant::now();
 
+    // If we stripped a prefix, clone req with the effective model so
+    // the adapter sees what the upstream provider expects (OpenRouter
+    // wants "openai/gpt-4o-mini", not "openrouter/openai/gpt-4o-mini").
+    let req_for_adapter: std::borrow::Cow<'_, ChatRequest> =
+        if effective_model == req.model {
+            std::borrow::Cow::Borrowed(&req)
+        } else {
+            let mut cloned = req.clone();
+            cloned.model = effective_model.clone();
+            std::borrow::Cow::Owned(cloned)
+        };
+
     let (resp, used_provider) = match provider.as_str() {
         "ollama" | "local" | "" => {
-            let r = ollama::chat(&state.ai_client, &req)
+            let r = ollama::chat(&state.ai_client, &*req_for_adapter)
                 .await
                 .map_err(|e| (StatusCode::BAD_GATEWAY, format!("ollama local: {e}")))?;
             (r, "ollama".to_string())
@@ -161,15 +283,52 @@ async fn chat(
                 StatusCode::SERVICE_UNAVAILABLE,
                 "OLLAMA_CLOUD_KEY not configured".to_string(),
             ))?;
-            let r = ollama_cloud::chat(key, &req)
+            let r = ollama_cloud::chat(key, &*req_for_adapter)
                 .await
                 .map_err(|e| (StatusCode::BAD_GATEWAY, format!("ollama cloud: {e}")))?;
             (r, "ollama_cloud".to_string())
         }
+        "openrouter" | "openrouter_free" => {
+            // Free-tier rescue rung. Added 2026-04-24 after iter 5
+            // repeated Ollama Cloud 502s on kimi-k2:1t — OpenRouter
+            // gives a different provider backbone as fallback.
+            let key = state.openrouter_key.as_deref().ok_or((
+                StatusCode::SERVICE_UNAVAILABLE,
+                "OPENROUTER_API_KEY not configured".to_string(),
+            ))?;
+            let r = openrouter::chat(key, &*req_for_adapter)
+                .await
+                .map_err(|e| (StatusCode::BAD_GATEWAY, format!("openrouter: {e}")))?;
+            (r, "openrouter".to_string())
+        }
+        "gemini" => {
+            // Phase 40 provider adapter. Google Generative Language
+            // API via query-string key auth (not bearer).
+            let key = state.gemini_key.as_deref().ok_or((
+                StatusCode::SERVICE_UNAVAILABLE,
+                "GEMINI_API_KEY not configured".to_string(),
+            ))?;
+            let r = gemini::chat(key, &*req_for_adapter)
+                .await
+                .map_err(|e| (StatusCode::BAD_GATEWAY, format!("gemini: {e}")))?;
+            (r, "gemini".to_string())
+        }
+        "claude" | "anthropic" => {
+            // Phase 40 provider adapter. Anthropic Messages API via
+            // x-api-key header + anthropic-version:2023-06-01.
+            let key = state.claude_key.as_deref().ok_or((
+                StatusCode::SERVICE_UNAVAILABLE,
+                "ANTHROPIC_API_KEY not configured".to_string(),
+            ))?;
+            let r = claude::chat(key, &*req_for_adapter)
+                .await
+                .map_err(|e| (StatusCode::BAD_GATEWAY, format!("claude: {e}")))?;
+            (r, "claude".to_string())
+        }
         other => {
             return Err((
                 StatusCode::BAD_REQUEST,
-                format!("unknown provider '{other}' — supported: ollama, ollama_cloud"),
+                format!("unknown provider '{other}' — supported: ollama, ollama_cloud, openrouter, gemini, claude"),
             ));
         }
     };

crates/gateway/src/v1/openrouter.rs

@@ -0,0 +1,217 @@
+//! OpenRouter adapter — free-tier rescue rung for /v1/chat.
+//!
+//! Direct HTTPS call to `https://openrouter.ai/api/v1/chat/completions`
+//! with Bearer auth. Mirrors the OpenAI-compatible shape so the model
+//! list can be expanded without code changes. Added 2026-04-24 after
+//! iter 5 hit repeated Ollama Cloud 502s on kimi-k2:1t — OpenRouter
+//! free-tier models give us a different provider backbone as fallback.
+//!
+//! Key sourcing priority:
+//!   1. Env var `OPENROUTER_API_KEY`
+//!   2. `/home/profit/.env`        (LLM Team convention)
+//!   3. `/root/llm_team_config.json` → providers.openrouter.api_key
+//!
+//! First hit wins. Key is resolved once at gateway startup and stored
+//! on `V1State.openrouter_key`.
+
+use std::time::Duration;
+use serde::{Deserialize, Serialize};
+
+use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
+
+const OR_BASE_URL: &str = "https://openrouter.ai/api/v1";
+const OR_TIMEOUT_SECS: u64 = 180;
+
+pub fn resolve_openrouter_key() -> Option<String> {
+    if let Ok(k) = std::env::var("OPENROUTER_API_KEY") {
+        if !k.trim().is_empty() { return Some(k.trim().to_string()); }
+    }
+    // LLM Team UI writes its key to ~/.env on the host user — pick it up
+    // from the same source so the free-tier rescue path works without
+    // an explicit systemd Environment= line.
+    for path in ["/home/profit/.env", "/root/.env"] {
+        if let Ok(raw) = std::fs::read_to_string(path) {
+            for line in raw.lines() {
+                if let Some(rest) = line.strip_prefix("OPENROUTER_API_KEY=") {
+                    let k = rest.trim().trim_matches('"').trim_matches('\'');
+                    if !k.is_empty() { return Some(k.to_string()); }
+                }
+            }
+        }
+    }
+    if let Ok(raw) = std::fs::read_to_string("/root/llm_team_config.json") {
+        if let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) {
+            if let Some(k) = v.pointer("/providers/openrouter/api_key").and_then(|x| x.as_str()) {
+                if !k.trim().is_empty() { return Some(k.trim().to_string()); }
+            }
+        }
+    }
+    None
+}
+
+pub async fn chat(
+    key: &str,
+    req: &ChatRequest,
+) -> Result<ChatResponse, String> {
+    // Strip the "openrouter/" prefix if the caller used the namespaced
+    // form so OpenRouter sees the raw model id (e.g. "openai/gpt-oss-120b:free").
+    let model = req.model.strip_prefix("openrouter/").unwrap_or(&req.model).to_string();
+
+    let body = ORChatBody {
+        model: model.clone(),
+        messages: req.messages.iter().map(|m| ORMessage {
+            role: m.role.clone(),
+            content: m.content.clone(),
+        }).collect(),
+        max_tokens: req.max_tokens.unwrap_or(800),
+        temperature: req.temperature.unwrap_or(0.3),
+        stream: false,
+    };
+
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(OR_TIMEOUT_SECS))
+        .build()
+        .map_err(|e| format!("build client: {e}"))?;
+
+    let t0 = std::time::Instant::now();
+    let resp = client
+        .post(format!("{}/chat/completions", OR_BASE_URL))
+        .bearer_auth(key)
+        // OpenRouter recommends Referer + Title for attribution; absent
+        // headers do not fail the call but help us see our traffic in
+        // their dashboard.
+        .header("HTTP-Referer", "https://vcp.devop.live")
+        .header("X-Title", "Lakehouse Scrum")
+        .json(&body)
+        .send()
+        .await
+        .map_err(|e| format!("openrouter.ai unreachable: {e}"))?;
+
+    let status = resp.status();
+    if !status.is_success() {
+        let body = resp.text().await.unwrap_or_else(|_| "?".into());
+        return Err(format!("openrouter.ai {}: {}", status, body));
+    }
+
+    let parsed: ORChatResponse = resp.json().await
+        .map_err(|e| format!("invalid openrouter response: {e}"))?;
+
+    let latency_ms = t0.elapsed().as_millis();
+    let choice = parsed.choices.into_iter().next()
+        .ok_or_else(|| "openrouter returned no choices".to_string())?;
+    let text = choice.message.content;
+
+    let prompt_tokens = parsed.usage.as_ref().map(|u| u.prompt_tokens).unwrap_or_else(|| {
+        let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
+        ((chars + 3) / 4) as u32
+    });
+    let completion_tokens = parsed.usage.as_ref().map(|u| u.completion_tokens).unwrap_or_else(|| {
+        ((text.chars().count() + 3) / 4) as u32
+    });
+
+    tracing::info!(
+        target: "v1.chat",
+        provider = "openrouter",
+        model = %model,
+        prompt_tokens,
+        completion_tokens,
+        latency_ms = latency_ms as u64,
+        "openrouter chat completed",
+    );
+
+    Ok(ChatResponse {
+        id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
+        object: "chat.completion",
+        created: chrono::Utc::now().timestamp(),
+        model,
+        choices: vec![Choice {
+            index: 0,
+            message: Message { role: "assistant".into(), content: text },
+            finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
+        }],
+        usage: UsageBlock {
+            prompt_tokens,
+            completion_tokens,
+            total_tokens: prompt_tokens + completion_tokens,
+        },
+    })
+}
+
+// -- OpenRouter wire shapes (OpenAI-compatible) --
+
+#[derive(Serialize)]
+struct ORChatBody {
+    model: String,
+    messages: Vec<ORMessage>,
+    max_tokens: u32,
+    temperature: f64,
+    stream: bool,
+}
+
+#[derive(Serialize)]
+struct ORMessage { role: String, content: String }
+
+#[derive(Deserialize)]
+struct ORChatResponse {
+    choices: Vec<ORChoice>,
+    #[serde(default)]
+    usage: Option<ORUsage>,
+}
+
+#[derive(Deserialize)]
+struct ORChoice {
+    message: ORMessageResp,
+    #[serde(default)]
+    finish_reason: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct ORMessageResp { content: String }
+
+#[derive(Deserialize)]
+struct ORUsage { prompt_tokens: u32, completion_tokens: u32 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resolve_openrouter_key_does_not_panic() {
+        // Smoke test — all three sources may or may not be set depending
+        // on environment; just confirm the call returns cleanly.
+        let _ = resolve_openrouter_key();
+    }
+
+    #[test]
+    fn chat_body_serializes_to_openai_shape() {
+        let body = ORChatBody {
+            model: "openai/gpt-oss-120b:free".into(),
+            messages: vec![
+                ORMessage { role: "user".into(), content: "review this".into() },
+            ],
+            max_tokens: 800,
+            temperature: 0.3,
+            stream: false,
+        };
+        let json = serde_json::to_string(&body).unwrap();
+        assert!(json.contains("\"model\":\"openai/gpt-oss-120b:free\""));
+        assert!(json.contains("\"messages\""));
+        assert!(json.contains("\"max_tokens\":800"));
+        assert!(json.contains("\"stream\":false"));
+    }
+
+    #[test]
+    fn model_prefix_strip_preserves_unprefixed() {
+        // If caller passes "openrouter/openai/gpt-oss-120b:free" we strip.
+        // If caller passes "openai/gpt-oss-120b:free" unchanged, we keep.
+        let cases = [
+            ("openrouter/openai/gpt-oss-120b:free", "openai/gpt-oss-120b:free"),
+            ("openai/gpt-oss-120b:free", "openai/gpt-oss-120b:free"),
+            ("google/gemma-3-27b-it:free", "google/gemma-3-27b-it:free"),
+        ];
+        for (input, expected) in cases {
+            let out = input.strip_prefix("openrouter/").unwrap_or(input);
+            assert_eq!(out, expected, "{input} should become {expected}");
+        }
+    }
+}

crates/gateway/src/v1/truth.rs

@@ -1,12 +1,10 @@
-use axum::{routing::get, Router};
 use serde::Serialize;
 use truth::default_truth_store;
 
-#[allow(dead_code)]
+// Note: truth_router() was a stub wrapper around a single /context route
-pub fn truth_router() -> Router {
+// that nothing called — v1/mod.rs wires get(truth::context) directly
-    Router::new()
+// onto its own router. Removed 2026-04-24 along with its #[allow(dead_code)]
-        .route("/context", get(context))
+// attribute; the handler below is the real surface.
-}
 
 #[derive(Serialize)]
 pub struct ContextResponse {

crates/ingestd/src/schema_evolution.rs

@@ -2,12 +2,9 @@
 /// When a source changes format (columns renamed, added, removed, type changed),
 /// the system detects the diff and can auto-map using AI or heuristic matching.
 
-#[allow(unused_imports)]
+use arrow::datatypes::{DataType, SchemaRef};
-use arrow::datatypes::{DataType, Schema, SchemaRef};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-#[allow(unused_imports)]
-use std::sync::Arc;
 
 /// A detected change between two schema versions.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -225,7 +222,8 @@ fn find_similar_column<'a>(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow::datatypes::Field;
+    use arrow::datatypes::{Field, Schema};
+    use std::sync::Arc;
 
     fn schema(fields: Vec<(&str, DataType)>) -> SchemaRef {
         Arc::new(Schema::new(

crates/ingestd/src/service.rs

@@ -1,10 +1,9 @@
-#[allow(unused_imports)]
 use axum::{
     Json, Router,
     extract::{Multipart, Path, Query, State},
     http::{HeaderMap, StatusCode},
     response::IntoResponse,
-    routing::{delete, get, patch, post},
+    routing::{get, post},
 };
 use bytes::Bytes;
 use object_store::ObjectStore;

crates/ingestd/src/watcher.rs

@@ -72,12 +72,14 @@ async fn process_inbox(
 
         let path = entry.path();
 
-        // Skip directories and hidden files
+        // Skip directories and hidden files. Bind filename once via
-        if path.is_dir() || path.file_name().map_or(true, |n| n.to_string_lossy().starts_with('.')) {
+        // let-else so the subsequent use is unwrap-free — previous
-            continue;
+        // version relied on a map_or guard above + an .unwrap() here
-        }
+        // being consistent, which is a fragile invariant.
-
+        if path.is_dir() { continue; }
-        let filename = path.file_name().unwrap().to_string_lossy().to_string();
+        let Some(fn_os) = path.file_name() else { continue; };
+        let filename = fn_os.to_string_lossy().to_string();
+        if filename.starts_with('.') { continue; }
         tracing::info!("watcher: found new file '{}'", filename);
 
         // Read file

crates/queryd/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2024"
 shared = { path = "../shared" }
 catalogd = { path = "../catalogd" }
 storaged = { path = "../storaged" }
+truth = { path = "../truth" }
 tokio = { workspace = true }
 axum = { workspace = true }
 serde = { workspace = true }

crates/queryd/src/delta.rs

@@ -84,14 +84,17 @@ pub async fn compact(
     // Load deltas
     let delta_batches = load_deltas(store, dataset_name).await?;
     let delta_count = delta_batches.len();
+    // Row counts captured before extend; previously base_rows subtracted delta_count (files) from rows — unit mismatch.
+    let base_row_count: usize = base_batches.iter().map(|b| b.num_rows()).sum();
+    let delta_row_count: usize = delta_batches.iter().map(|b| b.num_rows()).sum();
 
     let has_tombstones = !tombstones.is_empty();
     let nothing_to_do = delta_batches.is_empty() && !has_tombstones;
     if nothing_to_do {
         return Ok(CompactResult {
-            base_rows: base_batches.iter().map(|b| b.num_rows()).sum(),
+            base_rows: base_row_count,
             delta_rows: 0,
-            final_rows: base_batches.iter().map(|b| b.num_rows()).sum(),
+            final_rows: base_row_count,
             deltas_merged: 0,
             tombstones_applied: 0,
             rows_dropped_by_tombstones: 0,
@@ -99,7 +102,7 @@ pub async fn compact(
     }
 
     base_batches.extend(delta_batches);
-    let pre_filter_rows: usize = base_batches.iter().map(|b| b.num_rows()).sum();
+    let pre_filter_rows: usize = base_row_count + delta_row_count;
 
     // If primary key specified, deduplicate (keep last occurrence)
     let merged_batches = if let Some(_pk) = primary_key_col {
@@ -183,8 +186,8 @@ pub async fn compact(
     );
 
     Ok(CompactResult {
-        base_rows: pre_filter_rows - delta_count,  // rough base-before-deltas
+        base_rows: base_row_count,
-        delta_rows: delta_count,
+        delta_rows: delta_row_count,
         final_rows,
         deltas_merged: delta_count,
         tombstones_applied: tombstones.len(),

crates/queryd/src/service.rs

@@ -9,6 +9,9 @@ use axum::{
 };
 use serde::{Deserialize, Serialize};
 
+use std::sync::Arc;
+use truth::{RuleAction, TruthStore};
+
 use crate::context::QueryEngine;
 use crate::delta;
 use crate::paged::ResultStore;
@@ -17,12 +20,26 @@ use crate::paged::ResultStore;
 pub struct QueryState {
     pub engine: QueryEngine,
     pub result_store: ResultStore,
+    // Policy gate for incoming SQL. Every /sql and /paged request is
+    // evaluated against this store before hitting DataFusion. Added for
+    // P42-002 ("raw SQL forwarded without schema or policy gate") after
+    // the scrum master's queryd/service.rs finding looped across iters
+    // 3-5 without ever being reachable by the 6-line auto-applier.
+    pub truth: Arc<TruthStore>,
 }
 
 pub fn router(engine: QueryEngine) -> Router {
+    router_with_truth(engine, Arc::new(truth::sql_query_guard_store()))
+}
+
+/// Test/integration hook: construct the router with a caller-supplied
+/// TruthStore so tests can assert reject/pass behavior deterministically
+/// without depending on the default needle list.
+pub fn router_with_truth(engine: QueryEngine, truth: Arc<TruthStore>) -> Router {
     let state = QueryState {
         engine: engine.clone(),
         result_store: ResultStore::new(100, 50), // 100 rows/page, keep 50 results
+        truth,
     };
     Router::new()
         .route("/health", get(health))
@@ -52,6 +69,11 @@ struct QueryResponse {
     columns: Vec<ColumnInfo>,
     rows: serde_json::Value,
     row_count: usize,
+    // Elapsed wall time from handler entry to response. Required for
+    // audit-log parity — gateway's audit row previously stored null here.
+    // Scrum iter 9 finding, populated from std::time::Instant captured
+    // at the top of execute_query / paged_query.
+    latency_ms: u64,
 }
 
 #[derive(Serialize)]
@@ -71,12 +93,41 @@ fn batches_to_json(batches: &[RecordBatch]) -> Result<serde_json::Value, String>
     serde_json::from_slice(&buf).map_err(|e| format!("JSON parse error: {e}"))
 }
 
+/// Evaluate the request SQL against the configured TruthStore. Returns
+/// the Reject/Block message on the first failing mandatory rule so the
+/// handler can short-circuit. Returns None when all rules pass (or when
+/// the failures' declared action is non-mandatory like Redact/Pass).
+fn sql_policy_check(truth: &TruthStore, sql: &str) -> Option<String> {
+    let ctx = serde_json::json!({ "sql": sql });
+    for outcome in truth.evaluate("sql_query", &ctx) {
+        if !outcome.passed {
+            // FieldEmpty / FieldContainsAny etc. are enforced only when
+            // condition HOLDS (i.e. passed=true). Below means "passed=false",
+            // so the rule condition did not hold — no enforcement.
+            continue;
+        }
+        match &outcome.action {
+            RuleAction::Reject { message } | RuleAction::Block { message } => {
+                return Some(message.clone());
+            }
+            _ => {}
+        }
+    }
+    None
+}
+
 async fn execute_query(
     State(state): State<QueryState>,
     Json(req): Json<QueryRequest>,
 ) -> impl IntoResponse {
+    let started = std::time::Instant::now();
     tracing::info!("executing query: {}", req.sql);
 
+    if let Some(reason) = sql_policy_check(&state.truth, &req.sql) {
+        tracing::warn!("sql rejected by truth gate: {reason}");
+        return Err((StatusCode::FORBIDDEN, reason));
+    }
+
     match state.engine.query(&req.sql).await {
         Ok(batches) => {
             if batches.is_empty() {
@@ -84,6 +135,7 @@ async fn execute_query(
                     columns: vec![],
                     rows: serde_json::Value::Array(vec![]),
                     row_count: 0,
+                    latency_ms: started.elapsed().as_millis() as u64,
                 }));
             }
 
@@ -102,6 +154,7 @@ async fn execute_query(
                 columns,
                 rows,
                 row_count,
+                latency_ms: started.elapsed().as_millis() as u64,
             }))
         }
         Err(e) => Err((StatusCode::BAD_REQUEST, e)),
@@ -115,6 +168,10 @@ async fn paged_query(
     Json(req): Json<QueryRequest>,
 ) -> impl IntoResponse {
     tracing::info!("paged query: {}", req.sql);
+    if let Some(reason) = sql_policy_check(&state.truth, &req.sql) {
+        tracing::warn!("paged sql rejected by truth gate: {reason}");
+        return Err((StatusCode::FORBIDDEN, reason));
+    }
     match state.result_store.execute_and_store(&state.engine, &req.sql).await {
         Ok(handle) => Ok(Json(handle)),
         Err(e) => Err((StatusCode::BAD_REQUEST, e)),
@@ -211,3 +268,65 @@ async fn compact_dataset(
         Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
     }
 }
+
+#[cfg(test)]
+mod sql_policy_tests {
+    use super::*;
+    use truth::sql_query_guard_store;
+
+    // These tests exercise the policy gate without spinning up a DataFusion
+    // engine — they only need `TruthStore`. Purpose: prove the P42-002
+    // enforcement point actually rejects destructive SQL. This is the
+    // regression guard for the queryd/service.rs finding that looped
+    // across scrum iters 3-5.
+
+    #[test]
+    fn blocks_drop_table() {
+        let store = sql_query_guard_store();
+        let reason = sql_policy_check(&store, "DROP TABLE users").expect("must reject");
+        assert!(reason.contains("destructive"), "reason: {reason}");
+    }
+
+    #[test]
+    fn blocks_delete_from() {
+        let store = sql_query_guard_store();
+        assert!(sql_policy_check(&store, "delete from t where 1=1").is_some());
+    }
+
+    #[test]
+    fn blocks_truncate() {
+        let store = sql_query_guard_store();
+        assert!(sql_policy_check(&store, "TRUNCATE workers").is_some());
+    }
+
+    #[test]
+    fn blocks_empty_sql() {
+        let store = sql_query_guard_store();
+        assert!(sql_policy_check(&store, "").is_some());
+    }
+
+    #[test]
+    fn allows_benign_select() {
+        let store = sql_query_guard_store();
+        assert!(sql_policy_check(&store, "SELECT count(*) FROM workers").is_none());
+    }
+
+    #[test]
+    fn allows_select_with_deleted_word_in_column() {
+        // Substring match is narrow ("delete from", not "delete"), so a
+        // column named `deleted_at` doesn't trip the guard. Important
+        // check — false positives on benign queries would make the gate
+        // unusable in practice.
+        let store = sql_query_guard_store();
+        assert!(
+            sql_policy_check(&store, "SELECT deleted_at FROM t").is_none(),
+            "column names containing 'delete' must not be rejected"
+        );
+    }
+
+    #[test]
+    fn case_insensitive_match_catches_mixed_case() {
+        let store = sql_query_guard_store();
+        assert!(sql_policy_check(&store, "Drop Table X").is_some());
+    }
+}

crates/shared/src/lib.rs

@@ -4,3 +4,5 @@ pub mod arrow_helpers;
 pub mod config;
 pub mod pii;
 pub mod secrets;
+pub mod model_matrix;
+pub mod profiles;

crates/shared/src/model_matrix.rs

@@ -0,0 +1,69 @@
+//! Per-model token accounting. Entry point for the ModelMatrix work
+//! the aibridge `context::estimate_tokens` deprecation has been pointing
+//! at. Starts minimal — just `estimate_tokens` — so call sites can
+//! migrate off the deprecated helper. Extend with per-model context
+//! windows, max_tokens defaults, provider hints, etc. as we move the
+//! rest of `aibridge::context::known_windows` over.
+
+/// Namespace for per-model token + context accounting. Methods are
+/// associated functions — no instance required — because the underlying
+/// estimates are deterministic and stateless.
+pub struct ModelMatrix;
+
+impl ModelMatrix {
+    /// Rough token count — char count divided by 4, rounded up. This
+    /// is the same heuristic OpenAI's cookbook uses for English text;
+    /// it's within ±15% of BPE tokenizers for code + prose and doesn't
+    /// require a tokenizer lookup. Good enough for budget math where
+    /// the goal is "don't blow the context window" rather than exact
+    /// billing.
+    ///
+    /// Moved from `aibridge::context::estimate_tokens` (still there with
+    /// a `#[deprecated]` pointer — callers should migrate here). Empty
+    /// string → 0; one char → 1 (ceiling of 1/4 = 1).
+    pub fn estimate_tokens(text: &str) -> usize {
+        (text.chars().count() + 3) / 4
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ModelMatrix;
+
+    #[test]
+    fn empty_string_is_zero_tokens() {
+        assert_eq!(ModelMatrix::estimate_tokens(""), 0);
+    }
+
+    #[test]
+    fn three_chars_is_one_token() {
+        // 3 → ceil(3/4) = 1. Matches the deprecated helper's behavior
+        // so the migration is a drop-in replacement.
+        assert_eq!(ModelMatrix::estimate_tokens("abc"), 1);
+    }
+
+    #[test]
+    fn four_chars_is_one_token() {
+        assert_eq!(ModelMatrix::estimate_tokens("abcd"), 1);
+    }
+
+    #[test]
+    fn five_chars_is_two_tokens() {
+        assert_eq!(ModelMatrix::estimate_tokens("abcde"), 2);
+    }
+
+    #[test]
+    fn counts_chars_not_bytes() {
+        // Multi-byte UTF-8 chars count as 1 char each — important for
+        // prompts with emoji or non-ASCII text. "héllo" is 5 chars
+        // (5 unicode scalars) → ceil(5/4) = 2 tokens, same as "hello".
+        assert_eq!(ModelMatrix::estimate_tokens("héllo"), 2);
+        assert_eq!(ModelMatrix::estimate_tokens("📚📚📚📚"), 1);  // 4 chars
+    }
+
+    #[test]
+    fn large_text_scales_linearly() {
+        assert_eq!(ModelMatrix::estimate_tokens(&"x".repeat(400)), 100);
+        assert_eq!(ModelMatrix::estimate_tokens(&"x".repeat(401)), 101);
+    }
+}

crates/shared/src/profiles/execution.rs

@@ -0,0 +1,14 @@
+//! ExecutionProfile — the Phase 41 rename of Phase 17's ModelProfile.
+//!
+//! Carries what's needed to RUN inference: model tag, dataset bindings,
+//! HNSW config, embed model, bucket binding. Today this is a type
+//! alias over `crate::types::ModelProfile` — the PRD's
+//! "Backward compat: ModelProfile still loads, aliased to
+//! ExecutionProfile" line, honored literally.
+//!
+//! When the migration off the old name finishes, this file can either
+//! absorb the full struct definition or continue as an alias. Callers
+//! should reference `ExecutionProfile` going forward; `ModelProfile`
+//! stays exported from `types` for on-disk schema compat.
+
+pub use crate::types::ModelProfile as ExecutionProfile;

crates/shared/src/profiles/memory.rs

@@ -0,0 +1,38 @@
+//! MemoryProfile — how the agent's execution memory is kept.
+//!
+//! Phase 41 decomposition: the Phase 19 playbook_memory + Phase 26
+//! successful_playbooks + Phase 45 doc_refs all need per-profile
+//! tuning. Rather than bolt those onto ExecutionProfile, they live
+//! here so a "thin" execution profile can reuse a "fat" memory
+//! profile and vice versa.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct MemoryProfile {
+    pub id: String,
+    #[serde(default)]
+    pub description: String,
+    /// Phase 19: ceiling for playbook_memory boost on retrieval. 0
+    /// disables the boost entirely.
+    #[serde(default = "default_boost_ceiling")]
+    pub playbook_boost_ceiling: f32,
+    /// Phase 26: max history entries retained before rotation.
+    #[serde(default = "default_history_cap")]
+    pub history_cap: usize,
+    /// Phase 45: stale threshold for doc_refs before drift check
+    /// fires (hours).
+    #[serde(default = "default_stale_hours")]
+    pub doc_stale_hours: u32,
+    /// Phase 28: auto-retire playbooks that fail 3+ consecutive runs.
+    #[serde(default = "default_true")]
+    pub auto_retire_on_failure: bool,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+}
+
+fn default_boost_ceiling() -> f32 { 0.35 }
+fn default_history_cap() -> usize { 1000 }
+fn default_stale_hours() -> u32 { 168 } // one week
+fn default_true() -> bool { true }

crates/shared/src/profiles/mod.rs

@@ -0,0 +1,28 @@
+//! Phase 41 profile types.
+//!
+//! The existing `ModelProfile` (Phase 17) is aliased as
+//! `ExecutionProfile` here — it continues to carry the model +
+//! bindings + HNSW config needed to run inference. Three new profile
+//! types land alongside: `RetrievalProfile`, `MemoryProfile`,
+//! `ObserverProfile` — each owns a distinct slice of what used to be
+//! bundled.
+//!
+//! Backward-compat rule (PRD Phase 41): existing `ModelProfile` on
+//! disk continues to deserialize unchanged. New fields on the new
+//! profile types are `#[serde(default)]` so old payloads load with
+//! empty defaults.
+//!
+//! These are the canonical shapes — downstream code converts via
+//! `From<ModelProfile> for ExecutionProfile` (they're the same struct
+//! today, just named differently) and constructs the other three
+//! as needed.
+
+pub mod execution;
+pub mod retrieval;
+pub mod memory;
+pub mod observer;
+
+pub use execution::ExecutionProfile;
+pub use memory::MemoryProfile;
+pub use observer::ObserverProfile;
+pub use retrieval::RetrievalProfile;

crates/shared/src/profiles/observer.rs

@@ -0,0 +1,38 @@
+//! ObserverProfile — how loudly the observer logs this workload.
+//!
+//! Phase 41 decomposition: the observer's alert thresholds, escalation
+//! cadence, and log retention need per-workload tuning. Hot-path
+//! staffing workflows want aggressive alerting; batch backfills want
+//! quieter. This profile is read by mcp-server/observer.ts at
+//! activation-time.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ObserverProfile {
+    pub id: String,
+    #[serde(default)]
+    pub description: String,
+    /// How many consecutive failures trigger a cluster escalation to
+    /// LLM Team `/v1/chat` (qwen3-coder:480b).
+    #[serde(default = "default_failure_cluster_size")]
+    pub failure_cluster_size: u32,
+    /// Minimum seconds between alert emails for the same sig_hash.
+    /// Prevents alert storms during a regression.
+    #[serde(default = "default_alert_cooldown")]
+    pub alert_cooldown_secs: u32,
+    /// Observer ring buffer size. Older events fall off when full.
+    #[serde(default = "default_ring_size")]
+    pub ring_size: usize,
+    /// Whether to forward events to external Langfuse
+    /// (/v1/langfuse_trace). Off by default.
+    #[serde(default)]
+    pub forward_to_langfuse: bool,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+}
+
+fn default_failure_cluster_size() -> u32 { 3 }
+fn default_alert_cooldown() -> u32 { 300 } // 5 minutes
+fn default_ring_size() -> usize { 2000 }

crates/shared/src/profiles/retrieval.rs

@@ -0,0 +1,52 @@
+//! RetrievalProfile — what + how the agent reaches into memory.
+//!
+//! Phase 41 decomposition: the old ModelProfile bundled "what dataset
+//! can I read" (bound_datasets) AND "how do I rank results"
+//! (hnsw_config) with the model tag. Retrieval concerns split out here
+//! so a profile can swap its retrieval strategy without re-activating
+//! the model.
+//!
+//! Fields chosen for what's actually varied per-workload today:
+//!   - `top_k` / `rerank_top_k` — how many hits to fetch + rerank
+//!   - `freshness_cutoff_days` — Phase 45 doc-drift uses this
+//!   - `boost_playbook_memory` — Phase 19 meta-index feedback
+//!   - `enforce_sensitivity_gates` — Phase 13 access-control integration
+//!
+//! All fields are `#[serde(default)]` so loading a profile file that
+//! predates Phase 41 works without migration.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct RetrievalProfile {
+    /// Unique id — slug form, separate namespace from ExecutionProfile.
+    pub id: String,
+    /// Free-text operator description.
+    #[serde(default)]
+    pub description: String,
+    /// Default top-K for /vectors/search + /vectors/hybrid.
+    #[serde(default = "default_top_k")]
+    pub top_k: u32,
+    /// How many of the top-K to pass through the reranker. 0 disables
+    /// reranking for this profile.
+    #[serde(default = "default_rerank_top_k")]
+    pub rerank_top_k: u32,
+    /// Don't consider playbooks / docs older than this (days). 0 or
+    /// absent = no freshness filter.
+    #[serde(default)]
+    pub freshness_cutoff_days: u32,
+    /// Phase 19: boost workers/results by playbook_memory similarity.
+    #[serde(default)]
+    pub boost_playbook_memory: bool,
+    /// Phase 13: apply access-control masking on sensitive columns.
+    /// Default on — safety-first.
+    #[serde(default = "default_true")]
+    pub enforce_sensitivity_gates: bool,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+}
+
+fn default_top_k() -> u32 { 10 }
+fn default_rerank_top_k() -> u32 { 5 }
+fn default_true() -> bool { true }

crates/truth/Cargo.toml

@@ -8,3 +8,4 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
+toml = { workspace = true }

crates/truth/src/devops.rs

@@ -0,0 +1,49 @@
+//! DevOps task-class rules — scaffold for the long-horizon phase.
+//!
+//! Phase 42 PRD: "Terraform/Ansible rule shapes are scaffolded but
+//! unpopulated until the long-horizon phase. Keeps the dispatcher
+//! signature stable so no refactor needed later."
+//!
+//! This module is intentionally minimal. It registers no rules yet.
+//! The `devops_rules` function exists so callers can compose it onto
+//! a store (e.g. `devops_rules(staffing_rules(TruthStore::new()))`)
+//! without branching on whether the DevOps phase has landed.
+//!
+//! When the long-horizon phase fleshes out the DevOps rule set, the
+//! implementations drop in here — same `RuleCondition` primitives, same
+//! `TruthStore::evaluate` contract, zero upstream refactor.
+
+use crate::TruthStore;
+
+/// Register DevOps rules on the store. Currently a no-op scaffold —
+/// no rules are added. Safe to compose with other rule-set functions.
+///
+/// Planned task classes (not yet populated):
+///   - `devops.terraform_plan` — `terraform validate` + pre-plan
+///     sanity checks (no destroys without confirm flag, etc.)
+///   - `devops.ansible_playbook` — `ansible-lint` + privileged-task
+///     gates (no `become: true` on untagged hosts)
+///   - `devops.shell_command` — whitelist / blocklist for
+///     AI-generated shell invocations (covers what Phase 42
+///     queryd SQL gate does for SQL — same idea, shell surface)
+pub fn devops_rules(store: TruthStore) -> TruthStore {
+    // Intentionally empty. See module-level doc for the phased rollout.
+    store
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn devops_rules_is_a_noop_for_now() {
+        // Scaffold guarantee: composing devops_rules onto an empty
+        // store must not add any rules. Future long-horizon work will
+        // populate this and the assertion shifts to counting the
+        // expected additions.
+        let store = devops_rules(TruthStore::new());
+        assert_eq!(store.get_rules("devops.terraform_plan").len(), 0);
+        assert_eq!(store.get_rules("devops.ansible_playbook").len(), 0);
+        assert_eq!(store.get_rules("devops.shell_command").len(), 0);
+    }
+}

crates/truth/src/lib.rs

@@ -1,6 +1,10 @@
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
+pub mod staffing;
+pub mod devops;
+pub mod loader;
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct TruthRule {
     pub id: String,
@@ -18,6 +22,11 @@ pub enum RuleCondition {
     FieldMismatch { field: String, value: String },
     FieldEmpty { field: String },
     FieldGreater { field: String, threshold: i64 },
+    // Case-insensitive substring scan — true if the field value contains
+    // ANY of `needles`. Added for SQL/command guards where rules of the
+    // form "sql must not contain DROP/DELETE/TRUNCATE" need to express
+    // enforcement as a passing precondition being absent.
+    FieldContainsAny { field: String, needles: Vec<String> },
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -46,6 +55,15 @@ impl TruthStore {
             .push(rule);
     }
 
+    /// All rule IDs across every task class. Used by the file loader
+    /// to detect duplicate-ID collisions before registering new rules.
+    pub fn all_rule_ids(&self) -> std::collections::HashSet<String> {
+        self.rules
+            .values()
+            .flat_map(|v| v.iter().map(|r| r.id.clone()))
+            .collect()
+    }
+
     pub fn get_rules(&self, task_class: &str) -> Vec<&TruthRule> {
         self.rules
             .get(task_class)
@@ -124,6 +142,15 @@ fn evaluate_condition(cond: &RuleCondition, ctx: &serde_json::Value) -> bool {
                 .map(|n| n > *threshold)
                 .unwrap_or(false)
         }
+        RuleCondition::FieldContainsAny { field, needles } => {
+            match field_as_string(ctx, field) {
+                None => false,
+                Some(s) => {
+                    let haystack = s.to_ascii_lowercase();
+                    needles.iter().any(|n| haystack.contains(&n.to_ascii_lowercase()))
+                }
+            }
+        }
     }
 }
 
@@ -147,67 +174,64 @@ fn field_as_string(ctx: &serde_json::Value, path: &str) -> Option<String> {
     })
 }
 
-pub fn default_truth_store() -> TruthStore {
+/// Minimal SQL guard — rejects destructive verbs (DROP/TRUNCATE/DELETE).
+/// queryd/src/service.rs loads this into its `QueryState` and evaluates
+/// every `/sql` request against it before hitting the DataFusion engine.
+/// This is the P42-002 enforcement point flagged across scrum iters 3-5
+/// ("raw SQL forwarded without schema or policy gate").
+///
+/// Intentionally narrow: it's a safety net, not a full SQL parser. If
+/// callers need richer AST-aware enforcement they should extend this with
+/// structured rules rather than new needles.
+pub fn sql_query_guard_store() -> TruthStore {
     let mut store = TruthStore::new();
-
     store.add_rule(TruthRule {
-        id: "worker-active".to_string(),
+        id: "no-destructive-sql".to_string(),
-        task_class: "staffing.fill".to_string(),
+        task_class: "sql_query".to_string(),
-        description: "Worker must be active".to_string(),
+        description: "SQL must not contain destructive verbs".to_string(),
-        condition: RuleCondition::FieldEquals {
+        condition: RuleCondition::FieldContainsAny {
-            field: "worker.status".to_string(),
+            field: "sql".to_string(),
-            value: "active".to_string(),
+            needles: vec![
-        },
+                "drop table".to_string(),
-        action: RuleAction::Pass,
+                "drop schema".to_string(),
-    });
+                "drop database".to_string(),
-
+                "truncate".to_string(),
-    store.add_rule(TruthRule {
+                "delete from".to_string(),
-        id: "client-not-blacklisted".to_string(),
+            ],
-        task_class: "staffing.fill".to_string(),
-        description: "Worker cannot be blacklisted for client".to_string(),
-        condition: RuleCondition::FieldEquals {
-            field: "worker.client_blacklisted".to_string(),
-            value: "false".to_string(),
-        },
-        action: RuleAction::Pass,
-    });
-
-    store.add_rule(TruthRule {
-        id: "deadline-required".to_string(),
-        task_class: "staffing.fill".to_string(),
-        description: "Contract must have deadline".to_string(),
-        condition: RuleCondition::FieldEmpty {
-            field: "contract.deadline".to_string(),
         },
         action: RuleAction::Reject {
-            message: "Contract deadline is required".to_string(),
+            message: "destructive SQL rejected by truth.sql_query_guard".to_string(),
         },
     });
-
     store.add_rule(TruthRule {
-        id: "budget-required".to_string(),
+        id: "sql-not-empty".to_string(),
-        task_class: "staffing.fill".to_string(),
+        task_class: "sql_query".to_string(),
-        description: "Budget must be non-negative".to_string(),
+        description: "SQL must not be empty".to_string(),
-        condition: RuleCondition::FieldGreater {
+        condition: RuleCondition::FieldEmpty {
-            field: "contract.budget_per_hour_max".to_string(),
+            field: "sql".to_string(),
-            threshold: 0,
         },
-        action: RuleAction::Pass,
+        action: RuleAction::Reject {
-    });
+            message: "empty SQL rejected".to_string(),
-
-    store.add_rule(TruthRule {
-        id: "pii-redact".to_string(),
-        task_class: "staffing.any".to_string(),
-        description: "Redact PII before cloud calls".to_string(),
-        condition: RuleCondition::Always,
-        action: RuleAction::Redact {
-            fields: vec!["ssn".to_string(), "salary".to_string()],
         },
     });
-
     store
 }
 
+/// Phase 42 default store: staffing rules + DevOps scaffold composed
+/// onto an empty TruthStore. Per the PRD: "Staffing rules ship first;
+/// Terraform/Ansible rule shapes are scaffolded but unpopulated until
+/// the long-horizon phase." The composition order is irrelevant here
+/// (DevOps is empty) but preserved so the shape matches the PRD's
+/// expected "compose on top" pattern.
+///
+/// Moved out of inline in-function rule registration (2026-04-24) to
+/// land the Phase 42 module split the PRD called for: `staffing.rs` +
+/// `devops.rs` each owns their task-class rule sets. Behavior unchanged
+/// for existing callers.
+pub fn default_truth_store() -> TruthStore {
+    devops::devops_rules(staffing::staffing_rules(TruthStore::new()))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -520,4 +544,67 @@ mod tests {
         let actions = s.check("t");
         assert_eq!(actions.len(), 3, "check returns one action per rule regardless of condition");
     }
+
+    fn sql_guard_store() -> TruthStore {
+        let mut s = TruthStore::new();
+        s.add_rule(TruthRule {
+            id: "no-destructive".into(),
+            task_class: "sql_query".into(),
+            description: "SQL must not contain destructive verbs".into(),
+            condition: RuleCondition::FieldContainsAny {
+                field: "sql".into(),
+                needles: vec![
+                    "drop table".into(),
+                    "drop schema".into(),
+                    "truncate".into(),
+                    "delete from".into(),
+                ],
+            },
+            action: RuleAction::Reject {
+                message: "destructive SQL rejected".into(),
+            },
+        });
+        s
+    }
+
+    #[test]
+    fn field_contains_any_matches_case_insensitively() {
+        let s = sql_guard_store();
+        let ctx = serde_json::json!({"sql": "SELECT * FROM t; DROP TABLE users;"});
+        let o = s.evaluate("sql_query", &ctx);
+        assert!(o[0].passed, "condition holds when needle present (case-insensitive)");
+    }
+
+    #[test]
+    fn field_contains_any_is_false_when_no_needle_matches() {
+        let s = sql_guard_store();
+        let ctx = serde_json::json!({"sql": "SELECT count(*) FROM workers"});
+        let o = s.evaluate("sql_query", &ctx);
+        assert!(!o[0].passed, "benign SELECT should not match destructive needles");
+    }
+
+    #[test]
+    fn field_contains_any_false_when_field_missing() {
+        let s = sql_guard_store();
+        let ctx = serde_json::json!({});
+        let o = s.evaluate("sql_query", &ctx);
+        assert!(!o[0].passed, "missing field → condition cannot hold");
+    }
+
+    #[test]
+    fn field_contains_any_empty_needles_list_never_matches() {
+        let mut s = TruthStore::new();
+        s.add_rule(TruthRule {
+            id: "empty".into(),
+            task_class: "x".into(),
+            description: "".into(),
+            condition: RuleCondition::FieldContainsAny {
+                field: "sql".into(),
+                needles: vec![],
+            },
+            action: RuleAction::Pass,
+        });
+        let o = s.evaluate("x", &serde_json::json!({"sql": "anything"}));
+        assert!(!o[0].passed, "no needles → any::<bool> is false");
+    }
 }

crates/truth/src/loader.rs

@@ -0,0 +1,187 @@
+//! File-backed TruthRule loader (Phase 42 PRD).
+//!
+//! PRD: "truth/ dir at repo root — rule files, versioned in git."
+//! This module walks a directory, parses every `*.toml` file it finds,
+//! and registers the rules into a caller-supplied store. Rule IDs must
+//! be unique across the combined set — duplicate-ID collisions are
+//! load-time errors.
+//!
+//! The TOML format matches the shape at `truth/README.md`. The same
+//! `RuleCondition` + `RuleAction` enums used by the in-code registrars
+//! deserialize directly from `condition = { type = "FieldEquals", ... }`
+//! thanks to `#[serde(tag = "type")]`.
+
+use std::fs;
+use std::path::Path;
+use serde::Deserialize;
+
+use crate::{TruthRule, TruthStore};
+
+/// Deserialization wrapper — a TOML file is a list of [[rule]] blocks.
+#[derive(Deserialize)]
+struct RuleFile {
+    #[serde(default)]
+    rule: Vec<TruthRule>,
+}
+
+/// Load every `*.toml` file in `dir` and add its rules to `store`.
+/// Returns the number of rules loaded across all files.
+///
+/// Errors:
+///   - directory doesn't exist or can't be read
+///   - any `.toml` file fails to parse
+///   - any rule ID collides with an existing rule (same ID already
+///     registered in the store)
+///
+/// Non-goals: recursive walk (flat dir only), hot reload (one-shot load).
+pub fn load_from_dir(store: &mut TruthStore, dir: impl AsRef<Path>) -> Result<usize, String> {
+    let dir = dir.as_ref();
+    let entries = fs::read_dir(dir)
+        .map_err(|e| format!("read_dir {}: {e}", dir.display()))?;
+
+    let mut loaded_ids = store.all_rule_ids();
+    let mut count = 0usize;
+
+    let mut paths: Vec<_> = entries
+        .filter_map(|e| e.ok())
+        .map(|e| e.path())
+        .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("toml"))
+        .collect();
+    // Deterministic order — alphabetical by filename. Matters when a
+    // cross-file ID collision happens; the earlier filename wins
+    // nothing (both error), but the error message is reproducible.
+    paths.sort();
+
+    for path in paths {
+        let raw = fs::read_to_string(&path)
+            .map_err(|e| format!("read {}: {e}", path.display()))?;
+        let file: RuleFile = toml::from_str(&raw)
+            .map_err(|e| format!("parse {}: {e}", path.display()))?;
+        for rule in file.rule {
+            if !loaded_ids.insert(rule.id.clone()) {
+                return Err(format!(
+                    "duplicate rule id '{}' from {}",
+                    rule.id,
+                    path.display()
+                ));
+            }
+            store.add_rule(rule);
+            count += 1;
+        }
+    }
+
+    Ok(count)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+
+    fn write_file(dir: &Path, name: &str, content: &str) {
+        let path = dir.join(name);
+        let mut f = fs::File::create(&path).unwrap();
+        f.write_all(content.as_bytes()).unwrap();
+    }
+
+    #[test]
+    fn loads_rules_from_toml_files() {
+        let tmp = tempdir_for("loader_test");
+        write_file(&tmp, "a.toml", r#"
+[[rule]]
+id = "a-rule"
+task_class = "test"
+description = "test rule"
+action = { type = "Pass" }
+
+[rule.condition]
+type = "Always"
+"#);
+        let mut store = TruthStore::new();
+        let n = load_from_dir(&mut store, &tmp).unwrap();
+        assert_eq!(n, 1);
+        assert_eq!(store.get_rules("test").len(), 1);
+        let _ = fs::remove_dir_all(&tmp);
+    }
+
+    #[test]
+    fn rejects_duplicate_rule_ids() {
+        let tmp = tempdir_for("dup_ids");
+        write_file(&tmp, "a.toml", r#"
+[[rule]]
+id = "same"
+task_class = "t"
+description = ""
+action = { type = "Pass" }
+[rule.condition]
+type = "Always"
+"#);
+        write_file(&tmp, "b.toml", r#"
+[[rule]]
+id = "same"
+task_class = "t"
+description = ""
+action = { type = "Pass" }
+[rule.condition]
+type = "Always"
+"#);
+        let mut store = TruthStore::new();
+        let err = load_from_dir(&mut store, &tmp).unwrap_err();
+        assert!(err.contains("duplicate"), "got: {err}");
+        let _ = fs::remove_dir_all(&tmp);
+    }
+
+    #[test]
+    fn duplicate_with_in_code_rule_is_rejected() {
+        // Existing in-store IDs count as "already registered." Operator
+        // can't shadow an in-code rule by file without changing the ID.
+        let tmp = tempdir_for("dup_in_code");
+        write_file(&tmp, "conflict.toml", r#"
+[[rule]]
+id = "worker-active"
+task_class = "staffing.fill"
+description = "file attempt"
+action = { type = "Pass" }
+[rule.condition]
+type = "Always"
+"#);
+        // staffing_rules registers "worker-active"
+        let mut store = crate::staffing::staffing_rules(TruthStore::new());
+        let err = load_from_dir(&mut store, &tmp).unwrap_err();
+        assert!(err.contains("duplicate") && err.contains("worker-active"));
+        let _ = fs::remove_dir_all(&tmp);
+    }
+
+    #[test]
+    fn skips_non_toml_files() {
+        let tmp = tempdir_for("skip_non_toml");
+        write_file(&tmp, "a.toml", r#"
+[[rule]]
+id = "x"
+task_class = "t"
+description = ""
+action = { type = "Pass" }
+[rule.condition]
+type = "Always"
+"#);
+        write_file(&tmp, "README.md", "not a toml file");
+        let mut store = TruthStore::new();
+        let n = load_from_dir(&mut store, &tmp).unwrap();
+        assert_eq!(n, 1); // README.md ignored
+        let _ = fs::remove_dir_all(&tmp);
+    }
+
+    #[test]
+    fn missing_dir_returns_error() {
+        let mut store = TruthStore::new();
+        let err = load_from_dir(&mut store, "/nonexistent/path/here").unwrap_err();
+        assert!(err.contains("read_dir"));
+    }
+
+    fn tempdir_for(tag: &str) -> std::path::PathBuf {
+        let dir = std::env::temp_dir().join(format!("truth_loader_{}_{}", tag,
+            std::process::id()));
+        fs::create_dir_all(&dir).unwrap();
+        dir
+    }
+}

crates/truth/src/staffing.rs

@@ -0,0 +1,125 @@
+//! Staffing task-class rules for the TruthStore.
+//!
+//! Phase 42 PRD: "Staffing rules ship first. Terraform/Ansible rule
+//! shapes are scaffolded but unpopulated until the long-horizon phase."
+//! This module owns the staffing rule set; `devops.rs` holds the
+//! matching scaffold for the DevOps long-horizon.
+//!
+//! Rules registered here live under the task classes `staffing.fill`
+//! (fill proposals), `staffing.rescue` (rescue escalations), and
+//! `staffing.any` (rules that apply across all staffing task classes —
+//! PII redaction being the canonical example).
+//!
+//! All rules are evaluated via the `TruthStore::evaluate` walk, which
+//! pairs each rule's `RuleCondition` against a caller-supplied JSON
+//! context and emits a `RuleOutcome { passed, action }` per rule.
+//! Downstream enforcement (router gate, SQL gate, execution-loop gate)
+//! decides how to apply the action — `Reject` / `Block` shortcircuit,
+//! `Redact` mutates, `Pass` is informational.
+
+use crate::{RuleAction, RuleCondition, TruthRule, TruthStore};
+
+/// Register the staffing rule set on an existing store. Returns the
+/// store for chaining if the caller wants to fold other rule sets on
+/// top (e.g. `staffing_rules(devops_rules(TruthStore::new()))`).
+pub fn staffing_rules(mut store: TruthStore) -> TruthStore {
+    store.add_rule(TruthRule {
+        id: "worker-active".to_string(),
+        task_class: "staffing.fill".to_string(),
+        description: "Worker must be active".to_string(),
+        condition: RuleCondition::FieldEquals {
+            field: "worker.status".to_string(),
+            value: "active".to_string(),
+        },
+        action: RuleAction::Pass,
+    });
+
+    store.add_rule(TruthRule {
+        id: "client-not-blacklisted".to_string(),
+        task_class: "staffing.fill".to_string(),
+        description: "Worker cannot be blacklisted for client".to_string(),
+        condition: RuleCondition::FieldEquals {
+            field: "worker.client_blacklisted".to_string(),
+            value: "false".to_string(),
+        },
+        action: RuleAction::Pass,
+    });
+
+    store.add_rule(TruthRule {
+        id: "deadline-required".to_string(),
+        task_class: "staffing.fill".to_string(),
+        description: "Contract must have deadline".to_string(),
+        condition: RuleCondition::FieldEmpty {
+            field: "contract.deadline".to_string(),
+        },
+        action: RuleAction::Reject {
+            message: "Contract deadline is required".to_string(),
+        },
+    });
+
+    store.add_rule(TruthRule {
+        id: "budget-required".to_string(),
+        task_class: "staffing.fill".to_string(),
+        description: "Budget must be non-negative".to_string(),
+        condition: RuleCondition::FieldGreater {
+            field: "contract.budget_per_hour_max".to_string(),
+            threshold: 0,
+        },
+        action: RuleAction::Pass,
+    });
+
+    store.add_rule(TruthRule {
+        id: "pii-redact".to_string(),
+        task_class: "staffing.any".to_string(),
+        description: "Redact PII before cloud calls".to_string(),
+        condition: RuleCondition::Always,
+        action: RuleAction::Redact {
+            fields: vec!["ssn".to_string(), "salary".to_string()],
+        },
+    });
+
+    store
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn staffing_rules_registers_five_rules() {
+        // 4 staffing.fill rules + 1 staffing.any rule = 5 total.
+        // Regression guard: if someone adds a rule to this module
+        // without updating the count, this test surfaces it.
+        let store = staffing_rules(TruthStore::new());
+        let fill = store.get_rules("staffing.fill").len();
+        let any = store.get_rules("staffing.any").len();
+        assert_eq!(fill, 4);
+        assert_eq!(any, 1);
+    }
+
+    #[test]
+    fn blacklisted_worker_fails_the_rule() {
+        let store = staffing_rules(TruthStore::new());
+        let ctx = serde_json::json!({
+            "worker": { "client_blacklisted": "true" }
+        });
+        let outcomes = store.evaluate("staffing.fill", &ctx);
+        let blk = outcomes.iter().find(|o| o.rule_id == "client-not-blacklisted").unwrap();
+        assert!(!blk.passed, "blacklisted worker must fail the rule");
+    }
+
+    #[test]
+    fn missing_deadline_fires_reject_via_empty_condition() {
+        let store = staffing_rules(TruthStore::new());
+        // FieldEmpty passes when the field is missing — and the rule's
+        // action is Reject, so enforcement should fire.
+        let ctx = serde_json::json!({});
+        let outcomes = store.evaluate("staffing.fill", &ctx);
+        let deadline = outcomes.iter().find(|o| o.rule_id == "deadline-required").unwrap();
+        assert!(deadline.passed);
+        match &deadline.action {
+            RuleAction::Reject { message } => assert!(message.contains("deadline")),
+            _ => panic!("expected Reject action"),
+        }
+    }
+}

crates/ui/src/main.rs

@@ -1238,13 +1238,13 @@ fn IngestPanel() -> Element {
                         pg_tables.set(Some(tables));
                     }
                 }
-                Err(e) => pg_tables.set(None),
+                Err(_) => pg_tables.set(None),
             }
             pg_loading.set(false);
         });
     };
 
-    let mut import_table = move |table: String| {
+    let import_table = move |table: String| {
         let host = pg_host.read().clone();
         let db = pg_db.read().clone();
         spawn(async move {

crates/validator/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "validator"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }

crates/validator/src/devops.rs

@@ -0,0 +1,44 @@
+//! DevOps validator scaffold — long-horizon.
+//!
+//! PRD: "scaffold only: stubbed Terraform/Ansible validators
+//! (`terraform validate`, `ansible-lint`) for the long-horizon phase."
+//! Shipped as Unimplemented stubs so the execution-loop dispatcher
+//! has a consistent failure shape to surface ("phase 43 not wired")
+//! instead of a missing-impl panic.
+
+use crate::{Artifact, Report, Validator, ValidationError};
+
+pub struct TerraformValidator;
+
+impl Validator for TerraformValidator {
+    fn name(&self) -> &'static str { "devops.terraform" }
+    fn validate(&self, _artifact: &Artifact) -> Result<Report, ValidationError> {
+        Err(ValidationError::Unimplemented { artifact: "terraform_plan" })
+    }
+}
+
+pub struct AnsibleValidator;
+
+impl Validator for AnsibleValidator {
+    fn name(&self) -> &'static str { "devops.ansible" }
+    fn validate(&self, _artifact: &Artifact) -> Result<Report, ValidationError> {
+        Err(ValidationError::Unimplemented { artifact: "ansible_playbook" })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn terraform_scaffold_returns_unimplemented() {
+        let r = TerraformValidator.validate(&Artifact::TerraformPlan(serde_json::json!({})));
+        assert!(matches!(r, Err(ValidationError::Unimplemented { .. })));
+    }
+
+    #[test]
+    fn ansible_scaffold_returns_unimplemented() {
+        let r = AnsibleValidator.validate(&Artifact::AnsiblePlaybook(serde_json::json!({})));
+        assert!(matches!(r, Err(ValidationError::Unimplemented { .. })));
+    }
+}

crates/validator/src/lib.rs

@@ -0,0 +1,95 @@
+//! Phase 43 Validation Pipeline.
+//!
+//! PRD: "Staffing outputs run through schema / completeness /
+//! consistency / policy gates. Plug into Layer 5 execution loop —
+//! failure triggers observer-correction iteration."
+//!
+//! This crate provides the `Validator` trait + `Artifact` enum +
+//! Report/ValidationError types. Staffing validators (fill, email,
+//! playbook) and the DevOps scaffold live in submodules.
+//!
+//! Landed 2026-04-24 as a scaffold — the trait + types + module
+//! layout match the PRD; individual validator implementations are
+//! `Unimplemented` stubs that return a clear "phase 43 not wired"
+//! error rather than silently passing. The execution-loop integration
+//! (generate → validate → correct → retry) comes in a follow-up
+//! commit once the stubs are filled.
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+pub mod staffing;
+pub mod devops;
+
+/// What a validator saw. One variant per artifact class we validate.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "kind")]
+pub enum Artifact {
+    /// A fill proposal from the staffing executor — shape is
+    /// `{fills: [{candidate_id, name}]}` per PRD.
+    FillProposal(serde_json::Value),
+    /// An email/SMS draft for outreach.
+    EmailDraft(serde_json::Value),
+    /// A playbook being sealed for memory.
+    Playbook(serde_json::Value),
+    /// Terraform plan output (scaffold, long-horizon).
+    TerraformPlan(serde_json::Value),
+    /// Ansible playbook (scaffold, long-horizon).
+    AnsiblePlaybook(serde_json::Value),
+}
+
+/// Success report. Empty `findings` means a clean pass. Populated
+/// findings with `Severity::Warning` means "acceptable but notable" —
+/// the artifact passes. `Severity::Error` means validation failed;
+/// the validator should return `Err(...)` in that case, not `Ok`.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Report {
+    pub findings: Vec<Finding>,
+    pub elapsed_ms: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Finding {
+    pub field: String,
+    pub severity: Severity,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum Severity {
+    Warning,
+    Error,
+}
+
+/// Validation failure — what went wrong + where + why. Returned as
+/// `Err` from `validate`. Execution loop catches these and feeds them
+/// to the observer-correction retry loop.
+#[derive(Debug, Clone, Error, Serialize, Deserialize)]
+pub enum ValidationError {
+    /// Artifact schema doesn't match what we expected.
+    #[error("schema mismatch at {field}: {reason}")]
+    Schema { field: String, reason: String },
+    /// Required data missing (e.g. endorsed count != target count).
+    #[error("completeness: {reason}")]
+    Completeness { reason: String },
+    /// Data that's inconsistent with another source of truth
+    /// (e.g. worker_id doesn't exist in the workers table).
+    #[error("consistency: {reason}")]
+    Consistency { reason: String },
+    /// Policy violation — truth rule or access control said no.
+    #[error("policy: {reason}")]
+    Policy { reason: String },
+    /// Validator hasn't been implemented yet — scaffold stub.
+    #[error("validator not yet implemented for {artifact} — phase 43 scaffold")]
+    Unimplemented { artifact: &'static str },
+}
+
+/// Core validation contract. Implementations live in `staffing::*` and
+/// `devops::*`. The execution loop dispatches to the right impl based
+/// on the Artifact variant.
+pub trait Validator: Send + Sync {
+    fn validate(&self, artifact: &Artifact) -> Result<Report, ValidationError>;
+    /// Human-readable name for logs + Langfuse traces.
+    fn name(&self) -> &'static str;
+}

crates/validator/src/staffing/email.rs

@@ -0,0 +1,116 @@
+//! Email/SMS draft validator.
+//!
+//! PRD checks:
+//!   - Schema (TO/BODY fields present)
+//!   - Length (SMS ≤ 160 chars; email subject ≤ 78 chars)
+//!   - PII absence (no SSN / salary leaked into outgoing text)
+//!   - Worker-name consistency (name in message matches worker record)
+//!
+//! Scaffold implements schema + length. PII regex (SSN pattern,
+//! salary-number pattern) lives in `shared::pii::strip_pii` — plug in
+//! a follow-up when the validator caller knows the worker record to
+//! cross-check name consistency.
+
+use crate::{Artifact, Report, Validator, ValidationError};
+use std::time::Instant;
+
+pub struct EmailValidator;
+
+const SMS_MAX_CHARS: usize = 160;
+const EMAIL_SUBJECT_MAX_CHARS: usize = 78;
+
+impl Validator for EmailValidator {
+    fn name(&self) -> &'static str { "staffing.email" }
+
+    fn validate(&self, artifact: &Artifact) -> Result<Report, ValidationError> {
+        let started = Instant::now();
+        let value = match artifact {
+            Artifact::EmailDraft(v) => v,
+            other => return Err(ValidationError::Schema {
+                field: "artifact".into(),
+                reason: format!("EmailValidator expects EmailDraft, got {other:?}"),
+            }),
+        };
+
+        let to = value.get("to").and_then(|v| v.as_str()).ok_or(
+            ValidationError::Schema {
+                field: "to".into(),
+                reason: "missing or not a string".into(),
+            },
+        )?;
+        let body = value.get("body").and_then(|v| v.as_str()).ok_or(
+            ValidationError::Schema {
+                field: "body".into(),
+                reason: "missing or not a string".into(),
+            },
+        )?;
+
+        let is_sms = value.get("kind").and_then(|k| k.as_str()) == Some("sms");
+        if is_sms && body.len() > SMS_MAX_CHARS {
+            return Err(ValidationError::Completeness {
+                reason: format!("SMS body is {} chars, max {SMS_MAX_CHARS}", body.len()),
+            });
+        }
+
+        if let Some(subject) = value.get("subject").and_then(|v| v.as_str()) {
+            if subject.len() > EMAIL_SUBJECT_MAX_CHARS {
+                return Err(ValidationError::Completeness {
+                    reason: format!(
+                        "email subject is {} chars, max {EMAIL_SUBJECT_MAX_CHARS}",
+                        subject.len()
+                    ),
+                });
+            }
+        }
+
+        let _ = to; // touched for future name-consistency check
+        // TODO(phase-43 v2): PII scan + worker-name consistency.
+
+        Ok(Report {
+            findings: vec![],
+            elapsed_ms: started.elapsed().as_millis() as u64,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn long_sms_fails_completeness() {
+        let body = "x".repeat(200);
+        let r = EmailValidator.validate(&Artifact::EmailDraft(serde_json::json!({
+            "to": "+15555550123",
+            "body": body,
+            "kind": "sms"
+        })));
+        assert!(matches!(r, Err(ValidationError::Completeness { .. })));
+    }
+
+    #[test]
+    fn long_email_subject_fails_completeness() {
+        let r = EmailValidator.validate(&Artifact::EmailDraft(serde_json::json!({
+            "to": "a@b.com",
+            "body": "hi",
+            "subject": "x".repeat(100)
+        })));
+        assert!(matches!(r, Err(ValidationError::Completeness { .. })));
+    }
+
+    #[test]
+    fn missing_to_fails_schema() {
+        let r = EmailValidator.validate(&Artifact::EmailDraft(serde_json::json!({"body": "hi"})));
+        assert!(matches!(r, Err(ValidationError::Schema { field, .. }) if field == "to"));
+    }
+
+    #[test]
+    fn well_formed_email_passes() {
+        let r = EmailValidator.validate(&Artifact::EmailDraft(serde_json::json!({
+            "to": "hiring@example.com",
+            "subject": "Interview: Friday 10am",
+            "body": "Hi Jane — confirming interview Friday 10am."
+        })));
+        assert!(r.is_ok(), "well-formed email should pass: {:?}", r);
+    }
+}

crates/validator/src/staffing/fill.rs

@@ -0,0 +1,103 @@
+//! Fill-proposal validator.
+//!
+//! PRD checks:
+//!   - Schema compliance (propose_done shape matches
+//!     `{fills: [{candidate_id, name}]}`)
+//!   - Completeness (endorsed count == target_count)
+//!   - Worker existence (every candidate_id present in workers_500k)
+//!   - Status check (active, not_on_client_blacklist)
+//!   - Geo/role match (worker city/state/role matches contract)
+//!
+//! Today this is a scaffold — schema check is real (it's cheap); the
+//! worker-existence / status / geo checks need a catalog lookup and
+//! land in a follow-up when the catalog query helper is wired into
+//! this crate.
+
+use crate::{Artifact, Report, Validator, ValidationError};
+use std::time::Instant;
+
+pub struct FillValidator;
+
+impl Validator for FillValidator {
+    fn name(&self) -> &'static str { "staffing.fill" }
+
+    fn validate(&self, artifact: &Artifact) -> Result<Report, ValidationError> {
+        let started = Instant::now();
+        let value = match artifact {
+            Artifact::FillProposal(v) => v,
+            other => return Err(ValidationError::Schema {
+                field: "artifact".into(),
+                reason: format!("FillValidator expects FillProposal, got {other:?}"),
+            }),
+        };
+
+        // Schema check — the only real validation shipped in this
+        // scaffold. Catches the common "model emitted prose instead of
+        // JSON" failure mode before the consistency checks even run.
+        let fills = value.get("fills").and_then(|f| f.as_array()).ok_or(
+            ValidationError::Schema {
+                field: "fills".into(),
+                reason: "expected top-level `fills` array".into(),
+            },
+        )?;
+        for (i, fill) in fills.iter().enumerate() {
+            if fill.get("candidate_id").is_none() {
+                return Err(ValidationError::Schema {
+                    field: format!("fills[{i}].candidate_id"),
+                    reason: "missing".into(),
+                });
+            }
+            if fill.get("name").is_none() {
+                return Err(ValidationError::Schema {
+                    field: format!("fills[{i}].name"),
+                    reason: "missing".into(),
+                });
+            }
+        }
+
+        // TODO(phase-43 v2): worker-existence / status / geo checks.
+        // Need a catalog query handle injected into FillValidator's
+        // constructor — out of scope for the scaffold.
+
+        Ok(Report {
+            findings: vec![],
+            elapsed_ms: started.elapsed().as_millis() as u64,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn wrong_artifact_type_fails_schema() {
+        let r = FillValidator.validate(&Artifact::EmailDraft(serde_json::json!({})));
+        assert!(matches!(r, Err(ValidationError::Schema { .. })));
+    }
+
+    #[test]
+    fn missing_fills_array_fails_schema() {
+        let r = FillValidator.validate(&Artifact::FillProposal(serde_json::json!({})));
+        assert!(matches!(r, Err(ValidationError::Schema { field, .. }) if field == "fills"));
+    }
+
+    #[test]
+    fn fill_without_candidate_id_fails() {
+        let r = FillValidator.validate(&Artifact::FillProposal(serde_json::json!({
+            "fills": [{"name": "Jane"}]
+        })));
+        assert!(matches!(r, Err(ValidationError::Schema { field, .. }) if field.contains("candidate_id")));
+    }
+
+    #[test]
+    fn well_formed_proposal_passes_schema() {
+        let r = FillValidator.validate(&Artifact::FillProposal(serde_json::json!({
+            "fills": [
+                {"candidate_id": "W-123", "name": "Jane Doe"},
+                {"candidate_id": "W-456", "name": "John Smith"}
+            ]
+        })));
+        assert!(r.is_ok(), "well-formed proposal should pass schema: {:?}", r);
+    }
+}

crates/validator/src/staffing/mod.rs

@@ -0,0 +1,8 @@
+//! Staffing validators — fill proposals, email/SMS drafts, sealed
+//! playbooks. Phase 43 PRD: "the 0→85% pattern reproduces on real
+//! staffing tasks — the iteration loop with validation in place is
+//! what made small models successful."
+
+pub mod fill;
+pub mod email;
+pub mod playbook;

crates/validator/src/staffing/playbook.rs

@@ -0,0 +1,134 @@
+//! Sealed playbook validator.
+//!
+//! PRD checks:
+//!   - Operation format (`fill: Role xN in City, ST`)
+//!   - endorsed_names non-empty, ≤ target_count × 2
+//!   - fingerprint populated (Phase 25 validity window requirement)
+
+use crate::{Artifact, Report, Validator, ValidationError};
+use std::time::Instant;
+
+pub struct PlaybookValidator;
+
+impl Validator for PlaybookValidator {
+    fn name(&self) -> &'static str { "staffing.playbook" }
+
+    fn validate(&self, artifact: &Artifact) -> Result<Report, ValidationError> {
+        let started = Instant::now();
+        let value = match artifact {
+            Artifact::Playbook(v) => v,
+            other => return Err(ValidationError::Schema {
+                field: "artifact".into(),
+                reason: format!("PlaybookValidator expects Playbook, got {other:?}"),
+            }),
+        };
+
+        // Operation format: "fill: Role xN in City, ST" — at minimum
+        // we check the string-shape. Fuller grammar parse lives in
+        // phase 25 code where operations are structured beyond strings.
+        let op = value.get("operation").and_then(|v| v.as_str()).ok_or(
+            ValidationError::Schema {
+                field: "operation".into(),
+                reason: "missing or not a string".into(),
+            },
+        )?;
+        if !op.starts_with("fill:") {
+            return Err(ValidationError::Schema {
+                field: "operation".into(),
+                reason: format!("expected `fill: ...` prefix, got {op:?}"),
+            });
+        }
+
+        let endorsed = value.get("endorsed_names").and_then(|v| v.as_array()).ok_or(
+            ValidationError::Schema {
+                field: "endorsed_names".into(),
+                reason: "missing or not an array".into(),
+            },
+        )?;
+        if endorsed.is_empty() {
+            return Err(ValidationError::Completeness {
+                reason: "endorsed_names must be non-empty".into(),
+            });
+        }
+
+        if let Some(target) = value.get("target_count").and_then(|v| v.as_u64()) {
+            let max = (target * 2) as usize;
+            if endorsed.len() > max {
+                return Err(ValidationError::Completeness {
+                    reason: format!(
+                        "endorsed_names ({}) exceeds target_count × 2 ({max})",
+                        endorsed.len()
+                    ),
+                });
+            }
+        }
+
+        if value.get("fingerprint").and_then(|v| v.as_str()).map_or(true, |s| s.is_empty()) {
+            return Err(ValidationError::Schema {
+                field: "fingerprint".into(),
+                reason: "missing — required for Phase 25 validity window".into(),
+            });
+        }
+
+        Ok(Report {
+            findings: vec![],
+            elapsed_ms: started.elapsed().as_millis() as u64,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn well_formed_playbook_passes() {
+        let r = PlaybookValidator.validate(&Artifact::Playbook(serde_json::json!({
+            "operation": "fill: Welder x2 in Toledo, OH",
+            "endorsed_names": ["W-123", "W-456"],
+            "target_count": 2,
+            "fingerprint": "abc123"
+        })));
+        assert!(r.is_ok(), "got {:?}", r);
+    }
+
+    #[test]
+    fn empty_endorsed_names_fails_completeness() {
+        let r = PlaybookValidator.validate(&Artifact::Playbook(serde_json::json!({
+            "operation": "fill: Welder x2 in Toledo, OH",
+            "endorsed_names": [],
+            "fingerprint": "abc"
+        })));
+        assert!(matches!(r, Err(ValidationError::Completeness { .. })));
+    }
+
+    #[test]
+    fn overfull_endorsed_names_fails_completeness() {
+        let r = PlaybookValidator.validate(&Artifact::Playbook(serde_json::json!({
+            "operation": "fill: Welder x1 in Toledo, OH",
+            "endorsed_names": ["a", "b", "c"],
+            "target_count": 1,
+            "fingerprint": "abc"
+        })));
+        assert!(matches!(r, Err(ValidationError::Completeness { .. })));
+    }
+
+    #[test]
+    fn missing_fingerprint_fails_schema() {
+        let r = PlaybookValidator.validate(&Artifact::Playbook(serde_json::json!({
+            "operation": "fill: X x1 in A, B",
+            "endorsed_names": ["a"]
+        })));
+        assert!(matches!(r, Err(ValidationError::Schema { field, .. }) if field == "fingerprint"));
+    }
+
+    #[test]
+    fn wrong_operation_prefix_fails_schema() {
+        let r = PlaybookValidator.validate(&Artifact::Playbook(serde_json::json!({
+            "operation": "sms_draft: hello",
+            "endorsed_names": ["a"],
+            "fingerprint": "x"
+        })));
+        assert!(matches!(r, Err(ValidationError::Schema { .. })));
+    }
+}

crates/vectord/src/activation.rs

@@ -0,0 +1,118 @@
+//! Profile activation tracking (Phase 41 PRD).
+//!
+//! Phase 41 PRD called out `crates/vectord/src/activation.rs` with
+//! `ActivationTracker` + background-job pattern. The activation
+//! handler itself lives in `service.rs::activate_profile` (200+ lines
+//! of warm-up + bucket binding that's wired to VectorState); this
+//! module provides the type the PRD named and a single-flight guard
+//! that satisfies the PRD gate "refuse new activation if one is
+//! pending/running."
+//!
+//! Handler extraction (moving the body of `activate_profile` here)
+//! is deliberately NOT in this commit — it's a module-structure
+//! refactor, not a semantic change. When that lands, the inline
+//! `tokio::spawn` in `service.rs` moves into `ActivationTracker::start`
+//! and the HTTP handler shrinks to ~20 lines of validate + start +
+//! respond-202.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+/// Tracks in-flight profile activations. The PRD's "single-flight guard"
+/// lives here: callers check `is_pending` before starting a new activation
+/// and register via `mark_pending` if they proceed. On completion, they
+/// call `mark_complete` so the next caller can start.
+///
+/// Per-profile granularity — activating profile A doesn't block B.
+#[derive(Clone, Default)]
+pub struct ActivationTracker {
+    pending: Arc<RwLock<HashMap<String, String>>>, // profile_id → job_id
+}
+
+impl ActivationTracker {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Check if a profile has an activation already running. Returns the
+    /// in-flight job_id if so. Safe to call without holding a lock.
+    pub async fn is_pending(&self, profile_id: &str) -> Option<String> {
+        self.pending.read().await.get(profile_id).cloned()
+    }
+
+    /// Register a new activation as pending. Returns false if an
+    /// activation is already running for the same profile (caller should
+    /// return 409 Conflict or surface the existing job_id). Returns true
+    /// on successful registration.
+    pub async fn mark_pending(&self, profile_id: &str, job_id: &str) -> bool {
+        let mut guard = self.pending.write().await;
+        if guard.contains_key(profile_id) {
+            return false;
+        }
+        guard.insert(profile_id.to_string(), job_id.to_string());
+        true
+    }
+
+    /// Remove the pending marker when activation finishes (success OR
+    /// failure — both free the slot for the next caller).
+    pub async fn mark_complete(&self, profile_id: &str) {
+        self.pending.write().await.remove(profile_id);
+    }
+
+    /// How many activations are currently in-flight across all profiles.
+    pub async fn in_flight_count(&self) -> usize {
+        self.pending.read().await.len()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn empty_tracker_has_no_pending() {
+        let t = ActivationTracker::new();
+        assert_eq!(t.in_flight_count().await, 0);
+        assert!(t.is_pending("any-profile").await.is_none());
+    }
+
+    #[tokio::test]
+    async fn mark_pending_registers_the_job() {
+        let t = ActivationTracker::new();
+        assert!(t.mark_pending("profile-A", "job-1").await);
+        assert_eq!(t.in_flight_count().await, 1);
+        assert_eq!(t.is_pending("profile-A").await, Some("job-1".into()));
+    }
+
+    #[tokio::test]
+    async fn single_flight_guard_refuses_second_activation_same_profile() {
+        // PRD Phase 41 gate: "refuse new activation if one is
+        // pending/running." Same profile twice → second call returns
+        // false, caller must surface the in-flight job_id.
+        let t = ActivationTracker::new();
+        assert!(t.mark_pending("profile-A", "job-1").await);
+        assert!(!t.mark_pending("profile-A", "job-2").await);
+        // Still the first job — second registration didn't overwrite.
+        assert_eq!(t.is_pending("profile-A").await, Some("job-1".into()));
+    }
+
+    #[tokio::test]
+    async fn different_profiles_dont_block_each_other() {
+        // Per-profile granularity — activating A doesn't block B.
+        let t = ActivationTracker::new();
+        assert!(t.mark_pending("profile-A", "job-1").await);
+        assert!(t.mark_pending("profile-B", "job-2").await);
+        assert_eq!(t.in_flight_count().await, 2);
+    }
+
+    #[tokio::test]
+    async fn mark_complete_frees_the_slot() {
+        let t = ActivationTracker::new();
+        t.mark_pending("profile-A", "job-1").await;
+        t.mark_complete("profile-A").await;
+        assert_eq!(t.in_flight_count().await, 0);
+        // Next activation can now proceed.
+        assert!(t.mark_pending("profile-A", "job-2").await);
+    }
+}

crates/vectord/src/index_registry.rs

@@ -46,6 +46,21 @@ pub struct IndexMeta {
     /// Existing indexes: "W-", "CAND-", "W500K-", etc.
     #[serde(default)]
     pub id_prefix: Option<String>,
+    /// PRD 11.3 — when this index was last searched against. `None` =
+    /// never used since registration (or pre-field-existed metadata).
+    /// Incremental re-embed walks this to skip cold indexes.
+    /// Scrum iter 11 flagged the missing field as a UnitMismatch
+    /// because callers were reading `created_at` as a proxy for
+    /// liveness, which conflated "built" with "used."
+    #[serde(default)]
+    pub last_used: Option<DateTime<Utc>>,
+    /// PRD 11.3 — SHA-256 of (sorted source file list + chunk_size +
+    /// overlap + model_version). Lets incremental re-embed detect
+    /// "no change since last build" without scanning the source
+    /// Parquet. None = signature not computed yet (pre-existing
+    /// indexes before this field landed).
+    #[serde(default)]
+    pub build_signature: Option<String>,
 }
 
 fn default_bucket() -> String { "primary".to_string() }
@@ -128,4 +143,139 @@ impl IndexRegistry {
         self.indexes.write().await.remove(index_name);
         Ok(())
     }
+
+    /// Stamp `last_used = now()` on an index. Search handlers call this
+    /// on every hit so incremental re-embed (PRD 11.3) can tell live
+    /// indexes from cold ones. Silently no-ops if the index is unknown
+    /// — callers get best-effort behavior, not a 500 on a missing row.
+    pub async fn touch_used(&self, index_name: &str) {
+        if let Some(m) = self.indexes.write().await.get_mut(index_name) {
+            m.last_used = Some(Utc::now());
+        }
+    }
+}
+
+/// Compute a stable build_signature for PRD 11.3 incremental re-embed.
+/// Hashes (sorted source file list, chunk_size, overlap, model_version)
+/// so a caller can ask "has anything we built from changed?" without
+/// re-scanning the source parquet. Same inputs always produce the
+/// same hash.
+pub fn compute_build_signature(
+    source_files: &[impl AsRef<str>],
+    chunk_size: usize,
+    overlap: usize,
+    model_version: &str,
+) -> String {
+    use sha2::{Digest, Sha256};
+    let mut sorted: Vec<&str> = source_files.iter().map(|s| s.as_ref()).collect();
+    sorted.sort();
+    let mut hasher = Sha256::new();
+    for f in &sorted {
+        hasher.update(f.as_bytes());
+        hasher.update(b"\n");
+    }
+    hasher.update(chunk_size.to_le_bytes());
+    hasher.update(overlap.to_le_bytes());
+    hasher.update(model_version.as_bytes());
+    format!("{:x}", hasher.finalize())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_signature_is_deterministic() {
+        let sig1 = compute_build_signature(&["a.parquet", "b.parquet"], 800, 80, "v1");
+        let sig2 = compute_build_signature(&["a.parquet", "b.parquet"], 800, 80, "v1");
+        assert_eq!(sig1, sig2, "same inputs → same hash");
+    }
+
+    #[test]
+    fn build_signature_order_invariant() {
+        // Files get sorted internally so caller's order doesn't matter.
+        let sig_a = compute_build_signature(&["a.parquet", "b.parquet"], 800, 80, "v1");
+        let sig_b = compute_build_signature(&["b.parquet", "a.parquet"], 800, 80, "v1");
+        assert_eq!(sig_a, sig_b, "file list order must not affect hash");
+    }
+
+    #[test]
+    fn build_signature_changes_on_chunk_param() {
+        let sig_a = compute_build_signature(&["a.parquet"], 800, 80, "v1");
+        let sig_b = compute_build_signature(&["a.parquet"], 900, 80, "v1");
+        assert_ne!(sig_a, sig_b, "chunk_size change → different hash");
+    }
+
+    #[test]
+    fn build_signature_changes_on_model_version() {
+        let sig_a = compute_build_signature(&["a.parquet"], 800, 80, "v1");
+        let sig_b = compute_build_signature(&["a.parquet"], 800, 80, "v2");
+        assert_ne!(sig_a, sig_b, "model version change → different hash");
+    }
+
+    #[tokio::test]
+    async fn touch_used_updates_last_used() {
+        use object_store::memory::InMemory;
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let reg = IndexRegistry::new(store);
+        let meta = IndexMeta {
+            index_name: "test".into(),
+            source: "s".into(),
+            model_name: "m".into(),
+            model_version: "v1".into(),
+            dimensions: 768,
+            chunk_count: 0,
+            doc_count: 0,
+            chunk_size: 800,
+            overlap: 80,
+            storage_key: "k".into(),
+            created_at: Utc::now(),
+            build_time_secs: 0.0,
+            chunks_per_sec: 0.0,
+            bucket: "primary".into(),
+            vector_backend: Default::default(),
+            id_prefix: None,
+            last_used: None,
+            build_signature: None,
+        };
+        reg.register(meta).await.unwrap();
+        assert!(reg.get("test").await.unwrap().last_used.is_none());
+        reg.touch_used("test").await;
+        assert!(reg.get("test").await.unwrap().last_used.is_some());
+    }
+
+    #[tokio::test]
+    async fn touch_used_is_noop_on_missing_index() {
+        use object_store::memory::InMemory;
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let reg = IndexRegistry::new(store);
+        // No panic — unknown index just doesn't get touched.
+        reg.touch_used("nonexistent").await;
+    }
+
+    #[test]
+    fn index_meta_deserializes_without_new_fields_backcompat() {
+        // Pre-field-existence metadata files on disk must still load.
+        // Critical — we have ~40 .json meta files under vectors/meta/
+        // that predate these fields.
+        let json = r#"{
+            "index_name": "resumes_v1",
+            "source": "resumes",
+            "model_name": "nomic-embed-text",
+            "model_version": "latest",
+            "dimensions": 768,
+            "chunk_count": 100,
+            "doc_count": 10,
+            "chunk_size": 800,
+            "overlap": 80,
+            "storage_key": "vectors/resumes_v1.parquet",
+            "created_at": "2026-04-20T00:00:00Z",
+            "build_time_secs": 1.0,
+            "chunks_per_sec": 100.0
+        }"#;
+        let meta: IndexMeta = serde_json::from_str(json).expect("must deserialize pre-field meta");
+        assert!(meta.last_used.is_none());
+        assert!(meta.build_signature.is_none());
+        assert_eq!(meta.bucket, "primary");
+    }
 }

crates/vectord/src/lib.rs

@@ -7,7 +7,9 @@ pub mod harness;
 pub mod hnsw;
 pub mod index_registry;
 pub mod jobs;
+pub mod activation;
 pub mod playbook_memory;
+pub mod pathway_memory;
 pub mod doc_drift;
 pub mod promotion;
 pub mod refresh;

crates/vectord/src/playbook_memory.rs

@@ -1647,7 +1647,7 @@ mod validity_window_tests {
         let past = (chrono::Utc::now() - chrono::Duration::days(1)).to_rfc3339();
         let future = (chrono::Utc::now() + chrono::Duration::days(1)).to_rfc3339();
         let e_expired = mkentry("pb-expired", "Nashville", "TN", None, Some(past));
-        let e_alive = { let mut e = mkentry("pb-alive", "Nashville", "TN", None, Some(future)); e };
+        let e_alive = mkentry("pb-alive", "Nashville", "TN", None, Some(future));
         pm.set_entries(vec![e_expired, e_alive]).await.unwrap();
         let boosts = pm.compute_boost_for_filtered_with_role(
             &[1.0, 0.0, 0.0], 100, 0.5,

crates/vectord/src/promotion.rs

@@ -131,6 +131,11 @@ impl PromotionRegistry {
                 file.history.drain(0..drop);
             }
         }
+        // Bind `entry` ref-captured for the log line below so the log
+        // doesn't double-unwrap file.current — entry is Some-by-construction
+        // at the function boundary; past versions reached in via
+        // `.as_ref().unwrap()` twice, which compiled but would panic if
+        // the construction above ever changed.
         file.current = Some(entry);
         file.index_name = index_name.to_string();
 
@@ -140,10 +145,12 @@ impl PromotionRegistry {
         ops::put(&store, &key, json.into()).await?;
 
         self.cache.write().await.insert(index_name.to_string(), file.clone());
+        if let Some(cur) = &file.current {
             tracing::info!(
                 "promoted '{}' to config {:?} (trial={})",
-            index_name, file.current.as_ref().unwrap().config, file.current.as_ref().unwrap().trial_id,
+                index_name, cur.config, cur.trial_id,
             );
+        }
         Ok(file)
     }
 

crates/vectord/src/refresh.rs

@@ -308,6 +308,8 @@ async fn try_update_index_meta(
         bucket: "primary".to_string(),
         vector_backend: shared::types::VectorBackend::Parquet,
         id_prefix: None,
+        last_used: None,
+        build_signature: None,
     };
     index_registry.register(meta).await
 }

crates/vectord/src/service.rs

@@ -13,7 +13,7 @@ use std::sync::Arc;
 use aibridge::client::{AiClient, EmbedRequest, GenerateRequest};
 use catalogd::registry::Registry as CatalogRegistry;
 use storaged::registry::BucketRegistry;
-use crate::{agent, autotune, chunker, embedding_cache, harness, hnsw, index_registry, jobs, lance_backend, playbook_memory, promotion, rag, refresh, search, store, supervisor, trial};
+use crate::{agent, autotune, chunker, embedding_cache, harness, hnsw, index_registry, jobs, lance_backend, pathway_memory, playbook_memory, promotion, rag, refresh, search, store, supervisor, trial};
 use tokio::sync::Semaphore;
 
 #[derive(Clone)]
@@ -55,6 +55,11 @@ pub struct VectorState {
     /// and, when `use_playbook_memory` is set on /vectors/hybrid, boosts
     /// workers that were actually filled in semantically-similar past ops.
     pub playbook_memory: playbook_memory::PlaybookMemory,
+    /// Pathway memory — consensus-designed sidecar for full-context
+    /// backtracking + hot-swap of successful review pathways. See
+    /// crates/vectord/src/pathway_memory.rs for the design rationale
+    /// (10-probe N=3 ensemble, locked 2026-04-24).
+    pub pathway_memory: pathway_memory::PathwayMemory,
     /// Serializes embed calls from seed_playbook_memory to avoid
     /// concurrent socket collisions with the Python sidecar.
     pub embed_semaphore: Arc<Semaphore>,
@@ -78,6 +83,10 @@ pub fn router(state: VectorState) -> Router {
         .route("/indexes/{name}/bucket", axum::routing::patch(migrate_index_bucket))
         .route("/jobs", get(list_jobs))
         .route("/jobs/{id}", get(get_job))
+        // PRD Phase 41 alias — docs/CONTROL_PLANE_PRD.md specifies
+        // GET /vectors/profile/jobs/{id} for polling profile activations.
+        // Same handler as /jobs/{id}; the alias just matches the PRD URL.
+        .route("/profile/jobs/{id}", get(get_job))
         .route("/search", post(search_index))
         .route("/rag", post(rag_query))
         .route("/hybrid", post(hybrid_search))
@@ -137,6 +146,22 @@ pub fn router(state: VectorState) -> Router {
         // Phase 45 slice 3 — doc drift detection + human re-admission.
         .route("/playbook_memory/doc_drift/check/{id}", post(check_doc_drift))
         .route("/playbook_memory/doc_drift/resolve/{id}", post(resolve_doc_drift))
+        // Pathway memory — consensus-designed sidecar (2026-04-24).
+        // scrum_master_pipeline POSTs /pathway/insert at the end of each
+        // review, calls /pathway/query before running the ladder for a
+        // potential hot-swap, and posts /pathway/record_replay after a
+        // hot-swap succeeds or fails.
+        .route("/pathway/insert", post(pathway_insert))
+        .route("/pathway/query", post(pathway_query))
+        .route("/pathway/record_replay", post(pathway_record_replay))
+        .route("/pathway/stats", get(pathway_stats))
+        // ADR-021 Phase C: pre-review bug-fingerprint retrieval.
+        .route("/pathway/bug_fingerprints", post(pathway_bug_fingerprints))
+        // Mem0 ops (J 2026-04-25): upsert/retire/revise/history.
+        .route("/pathway/upsert", post(pathway_upsert))
+        .route("/pathway/retire", post(pathway_retire))
+        .route("/pathway/revise", post(pathway_revise))
+        .route("/pathway/history/{trace_uid}", get(pathway_history))
         .with_state(state)
 }
 
@@ -238,6 +263,8 @@ async fn create_index(
                     bucket: bucket.clone(),
                     vector_backend: shared::types::VectorBackend::Parquet,
                     id_prefix: None,
+                    last_used: None,
+                    build_signature: None,
                 };
                 let _ = registry.register(meta).await;
 
@@ -454,51 +481,6 @@ async fn copy_key(
     storaged::ops::put(dst, key, data).await
 }
 
-// --- unused legacy function below, kept for reference ---
-
-#[allow(dead_code)]
-/// Legacy single-pipeline embedding (replaced by supervisor).
-async fn _run_embedding_job_legacy(
-    job_id: &str,
-    index_name: &str,
-    chunks: &[chunker::TextChunk],
-    ai_client: &AiClient,
-    store: &Arc<dyn ObjectStore>,
-    tracker: &jobs::JobTracker,
-) -> Result<String, String> {
-    let batch_size = 32;
-    let mut all_vectors: Vec<Vec<f64>> = Vec::new();
-    let start = std::time::Instant::now();
-
-    for (i, batch) in chunks.chunks(batch_size).enumerate() {
-        let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
-
-        let embed_resp = ai_client.embed(EmbedRequest {
-            texts,
-            model: None,
-        }).await.map_err(|e| format!("embed batch {} error: {e}", i))?;
-
-        all_vectors.extend(embed_resp.embeddings);
-
-        // Update progress
-        let elapsed = start.elapsed().as_secs_f32();
-        let rate = if elapsed > 0.0 { all_vectors.len() as f32 / elapsed } else { 0.0 };
-        tracker.update_embed_progress(job_id, all_vectors.len(), rate).await;
-
-        // Log every 100 batches
-        if (i + 1) % 100 == 0 {
-            let pct = (all_vectors.len() as f32 / chunks.len() as f32) * 100.0;
-            let eta = if rate > 0.0 { (chunks.len() - all_vectors.len()) as f32 / rate } else { 0.0 };
-            tracing::info!("job {job_id}: {}/{} chunks ({pct:.0}%), {rate:.0}/sec, ETA {eta:.0}s",
-                all_vectors.len(), chunks.len());
-        }
-    }
-
-    // Store
-    let key = store::store_embeddings(store, index_name, chunks, &all_vectors).await?;
-    Ok(key)
-}
-
 // --- Job Status ---
 
 async fn list_jobs(State(state): State<VectorState>) -> impl IntoResponse {
@@ -1580,10 +1562,13 @@ async fn activate_profile(
         tracker.complete(&job_id, result).await;
     });
 
-    Ok(Json(json!({
+    // PRD Phase 41 gate: "Activate a profile → returns 202 in <100ms
+    // → job completes in background". 202 ACCEPTED signals async-work
+    // started; clients poll /vectors/jobs/{job_id} for progress.
+    Ok((StatusCode::ACCEPTED, Json(json!({
         "job_id": job_id_for_response,
         "message": format!("profile activation started — poll /vectors/jobs/{} for progress", job_id_for_response),
-    })))
+    }))))
 }
 
 /// Unload this profile's model and clear the active slot. No-op if the
@@ -2833,6 +2818,149 @@ async fn lance_build_scalar_index(
     }
 }
 
+// ─── Pathway memory handlers ──────────────────────────────────────────
+//
+// Thin wrappers around pathway_memory::PathwayMemory. HTTP surface is
+// deliberately small — four endpoints cover the full lifecycle:
+// insert at end-of-review, query before running the ladder,
+// record_replay after a hot-swap, and stats for the VCP UI.
+
+#[derive(Deserialize)]
+struct PathwayQueryRequest {
+    task_class: String,
+    file_path: String,
+    signal_class: Option<String>,
+    query_vec: Vec<f32>,
+}
+
+async fn pathway_insert(
+    State(state): State<VectorState>,
+    Json(trace): Json<pathway_memory::PathwayTrace>,
+) -> impl IntoResponse {
+    match state.pathway_memory.insert(trace).await {
+        Ok(()) => Ok(Json(json!({"ok": true}))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn pathway_query(
+    State(state): State<VectorState>,
+    Json(req): Json<PathwayQueryRequest>,
+) -> impl IntoResponse {
+    let cand = state
+        .pathway_memory
+        .query_hot_swap(
+            &req.task_class,
+            &req.file_path,
+            req.signal_class.as_deref(),
+            &req.query_vec,
+        )
+        .await;
+    // 200 with null candidate means "no hot-swap"; this is a normal
+    // path, not an error — callers should proceed with the full ladder.
+    Json(json!({ "candidate": cand }))
+}
+
+#[derive(Deserialize)]
+struct PathwayReplayRequest {
+    pathway_id: String,
+    succeeded: bool,
+}
+
+async fn pathway_record_replay(
+    State(state): State<VectorState>,
+    Json(req): Json<PathwayReplayRequest>,
+) -> impl IntoResponse {
+    match state
+        .pathway_memory
+        .record_replay_outcome(&req.pathway_id, req.succeeded)
+        .await
+    {
+        Ok(()) => Ok(Json(json!({"ok": true}))),
+        Err(e) => Err((StatusCode::NOT_FOUND, e)),
+    }
+}
+
+async fn pathway_stats(State(state): State<VectorState>) -> impl IntoResponse {
+    Json(state.pathway_memory.stats().await)
+}
+
+#[derive(Deserialize)]
+struct PathwayBugFingerprintsRequest {
+    task_class: String,
+    file_path: String,
+    signal_class: Option<String>,
+    limit: Option<usize>,
+}
+
+async fn pathway_bug_fingerprints(
+    State(state): State<VectorState>,
+    Json(req): Json<PathwayBugFingerprintsRequest>,
+) -> impl IntoResponse {
+    let fps = state
+        .pathway_memory
+        .bug_fingerprints_for(
+            &req.task_class,
+            &req.file_path,
+            req.signal_class.as_deref(),
+            req.limit.unwrap_or(5),
+        )
+        .await;
+    Json(json!({ "fingerprints": fps }))
+}
+
+// ─── Mem0 ops endpoints (J 2026-04-25) ───
+
+async fn pathway_upsert(
+    State(state): State<VectorState>,
+    Json(trace): Json<pathway_memory::PathwayTrace>,
+) -> impl IntoResponse {
+    match state.pathway_memory.upsert(trace).await {
+        Ok(outcome) => Ok(Json(json!({"ok": true, "outcome": outcome}))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct PathwayRetireRequest {
+    trace_uid: String,
+    reason: String,
+}
+
+async fn pathway_retire(
+    State(state): State<VectorState>,
+    Json(req): Json<PathwayRetireRequest>,
+) -> impl IntoResponse {
+    match state.pathway_memory.retire(&req.trace_uid, &req.reason).await {
+        Ok(touched) => Ok(Json(json!({"ok": true, "retired": touched}))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct PathwayReviseRequest {
+    parent_trace_uid: String,
+    new_trace: pathway_memory::PathwayTrace,
+}
+
+async fn pathway_revise(
+    State(state): State<VectorState>,
+    Json(req): Json<PathwayReviseRequest>,
+) -> impl IntoResponse {
+    match state.pathway_memory.revise(&req.parent_trace_uid, req.new_trace).await {
+        Ok(outcome) => Ok(Json(json!({"ok": true, "outcome": outcome}))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn pathway_history(
+    State(state): State<VectorState>,
+    axum::extract::Path(trace_uid): axum::extract::Path<String>,
+) -> impl IntoResponse {
+    let chain = state.pathway_memory.history(&trace_uid).await;
+    Json(json!({"trace_uid": trace_uid, "chain_len": chain.len(), "chain": chain}))
+}
+
 #[cfg(test)]
 mod extractor_tests {
     use super::*;

crates/vectord/src/trial.rs

@@ -10,7 +10,6 @@
 /// JSONL on every event. See `append_log.rs` for the full rationale.
 
 use chrono::{DateTime, Utc};
-use object_store::ObjectStore;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Arc;

docs/DECISIONS.md

@@ -99,3 +99,8 @@
 **Date:** 2026-04-19
 **Decision:** `catalogd::Registry::register(name, fingerprint, objects)` is idempotent on `name`. If no manifest for `name` exists, create one. If one exists with the same `schema_fingerprint`, reuse its `DatasetId`, replace `objects`, bump `updated_at`, and write through. If one exists with a different `schema_fingerprint`, reject with `409 Conflict` (HTTP) / `FAILED_PRECONDITION` (gRPC). A one-shot operator endpoint `POST /catalog/dedupe` collapses any pre-existing duplicates (preferring the manifest with a non-null `row_count`, then the most recently updated).
 **Rationale:** Registry was keyed by surrogate `DatasetId` with no uniqueness constraint on `name`, so every caller that re-registered (re-ingest, external cron, gRPC retry) silently created a parallel manifest pointing at the same parquet — accumulating 308× `successful_playbooks` in live state before detection. The fingerprint gate turns re-ingest into an explicit no-op (matching PRD invariant #5 "ingestd is idempotent — re-ingesting the same file is a no-op") while forcing schema drift to be visible instead of silently clobbering. 409 status separates policy rejections from server errors, which matters for the Phase 12 tool-consumer ecosystem. Concurrency: the write lock is held across the storage write to close the check→insert TOCTOU window; serializing registers is acceptable because registers-per-second is low on the ingest path. Audit: idempotent-register events are visible as bumps to the stored manifest's `updated_at` field and in `catalogd` tracing output (tracing is non-durable, operator view only); `DedupeReport` is the return-value audit for cleanup runs. No event-journal entries are emitted — ADR-012 scopes the journal to row-level mutations, not catalog-manifest operations.
+
+## ADR-021: Semantic-correctness layer on pathway_memory — matrix-indexed bug grammar
+**Date:** 2026-04-24
+**Decision:** Extend `pathway_memory::PathwayTrace` (ADR added 2026-04-24 in same commit as this one) with a semantic-correctness layer so the matrix index compounds recognition of unit/type/shape bugs across iterations. Three new fields: `semantic_flags: Vec<SemanticFlag>` (enum: `UnitMismatch`, `TypeConfusion`, `NullableConfusion`, `OffByOne`, `StaleReference`, `PseudoImpl`, `DeadCode`, `WarningNoise`, `BoundaryViolation`), `type_hints_used: Vec<TypeHint>` (schema/type context the reviewer was given — catalogd column types for SQL-touching code, Arrow `RecordBatch.schema()` accessors for Rust, Rust struct field types for everything else), and `bug_fingerprints: Vec<BugFingerprint>` (structural pattern hash, e.g. `{lhs_unit: "rows", rhs_unit: "files", op: "-"}` → stable SHA for similarity retrieval). Scrum pipeline pre-review: query matrix index for bug fingerprints flagged on this file's narrow fingerprint (same `task_class + file_prefix + signal_class` as hot-swap) and prepend them to the reviewer prompt as "watch for these patterns historically found here." Reviewer prompt explicitly tags each finding with a `semantic_flag`. `truth::evaluate()` gets a review-time task_class (`code_review.unit_check`) that consumes parsed-fact rules like `FieldContainsAny { field: "code_expression", needles: ["row_count - file_count", "bytes_read - row_count"] }` — the same primitive we use for SQL guard in P42-002.
+**Rationale:** The 2026-04-24 `queryd/src/delta.rs` `base_rows = pre_filter_rows - delta_count` bug (86901f8) was found by a human reading the code and noticing units didn't match. The hardened mechanical applier *cannot* catch this — its gates are syntactic (warning count, patch size, rationale-token alignment) not semantic. At 100 bugs this deep, no human catches them all; the signal→commit loop is capped by what humans can notice per iteration. We already ship the primitives: `catalogd` knows column types per dataset, Arrow `RecordBatch.schema()` is on every hot-path call, `truth::evaluate()` runs arbitrary field conditions at runtime, `shared/arrow_helpers` has typed row/byte/file accessors. All of this is used at RUNTIME; none is fed into the REVIEW pipeline. Semantic flags + bug fingerprints turn the matrix index from "what review happened" (current) into "what category of bug appeared where" (compounding) — so iter-20 scrum on `crates/queryd/src/` preempts review prompts with "this crate had a row/file unit mismatch in iter 7 (delta.rs:189); check every arithmetic on `*_count` variables." Non-goals: we are NOT building a full type-inference engine (reuse Rust's `rustdoc`-level type info for structs, Arrow's schema for RecordBatch, catalogd's column types for SQL — everything beyond is Phase 3). Non-goals: this is not a linter — clippy/rustc already catch syntactic issues; this catches SEMANTIC ones (same type, wrong units/role). Bootstrap path: start with the 9 `SemanticFlag` variants above; add new variants only when a bug is found that doesn't fit an existing one. Gate alignment with hot-swap: a pathway that repeatedly produces bugs of the same `SemanticFlag` variant on the same narrow fingerprint is more valuable as a "watch this file for X" signal than as a hot-swap candidate — retirement logic needs to consider both replay success_rate AND whether the pathway is serving as a bug-pattern beacon.

docs/SCRUM_MASTER_SPEC.md

@@ -0,0 +1,275 @@
+# Scrum Master Pipeline — Spec + Current State
+
+**Status:** Active iteration on branch `scrum/auto-apply-19814` → PR #11 at git.agentview.dev/profit/lakehouse
+**Last iter:** 9 (2026-04-24)
+**Branch commit head:** `f4cff66` (ADR-021 Phase D fix)
+
+This doc is the single handoff artifact for the scrum-master + auto-apply + pathway-memory loop built during 2026-04-24 sessions. A fresh Claude Code session reading this + `docs/DECISIONS.md` (ADR-020 and ADR-021 specifically) + `MEMORY.md` should have the same context as the session that wrote it.
+
+## 1. What the loop is
+
+An autonomous review-and-commit pipeline that:
+
+1. **Scrum master** (`tests/real-world/scrum_master_pipeline.ts`) — walks a target-file list, asks a 9-rung escalation ladder of cloud models to produce a forensic audit against PRD + a change proposal doc, retries with learning context until acceptance, emits a structured review row.
+2. **Pathway memory** (`crates/vectord/src/pathway_memory.rs`) — stores the full backtrack context of each review (attempts, KB chunks, flags, bug fingerprints) indexed by a narrow fingerprint (`task_class + file_prefix + signal_class`). On every new review, it prepends historical bug patterns as a preamble so the reviewer preempts recurrences. Retired pathways auto-exclude themselves from hot-swap eligibility.
+3. **Auto-applier** (`tests/real-world/scrum_applier.ts`) — filters schema_v4 review rows by gradient_tier + confidence, asks `qwen3-coder:480b` for concrete `old_string/new_string` patches, runs `cargo check --workspace`, commits on green OR reverts on red/warning-count-up/rationale-mismatch.
+4. **Observer** (`mcp-server/observer.ts`) — receives per-file `/event` emissions, escalates failure clusters to LLM Team via `/v1/chat` with `qwen3-coder:480b`.
+5. **Auditor** (`auditor/audit.ts`) — external N=3 consensus re-check of scrum findings; writes to `data/_kb/audit_facts.jsonl`.
+
+The guiding principle: **every KB write has a reader, every PR claim is diff-verifiable.**
+
+## 2. The 9-rung ladder (cloud-first, strongest-model-first)
+
+Defined in `tests/real-world/scrum_master_pipeline.ts` at `const LADDER`:
+
+| # | Provider | Model | Role |
+|---|---|---|---|
+| 1 | ollama_cloud | `kimi-k2:1t` | flagship, 1T params |
+| 2 | ollama_cloud | `qwen3-coder:480b` | coding specialist, 480B |
+| 3 | ollama_cloud | `deepseek-v3.1:671b` | reasoning, 671B |
+| 4 | ollama_cloud | `mistral-large-3:675b` | deep analysis, 675B |
+| 5 | ollama_cloud | `gpt-oss:120b` | reliable workhorse |
+| 6 | ollama_cloud | `qwen3.5:397b` | dense 397B, final thinker |
+| 7 | openrouter | `openai/gpt-oss-120b:free` | free-tier rescue |
+| 8 | openrouter | `google/gemma-3-27b-it:free` | fastest rescue |
+| 9 | ollama | `qwen3.5:latest` | last-resort local |
+
+**Each attempt is evaluated by `isAcceptable()`** (chars ≥ 3800 AND not a malformed JSON-only dump). On reject, the next rung sees a learning preamble with the prior rejection reason.
+
+## 3. Tree-split reducer
+
+Files larger than `FILE_TREE_SPLIT_THRESHOLD = 6000` bytes get chunked into `FILE_SHARD_SIZE = 3500`-byte shards. Each shard gets summarized via a fast rung, summaries are concatenated with internal `§N§` markers, then fed as a SCRATCHPAD to the reviewer. The `§N§` markers are stripped before the reviewer sees the merged context so it cannot claim "(shard 3)" in titles.
+
+Bug regime this fixed: pre-tree-split iters had reviewers claim fields were "missing" because the field was past the 6KB context cutoff, not actually absent.
+
+## 4. Schema v4 KB rows
+
+`data/_kb/scrum_reviews.jsonl` — one row per accepted review. Fields:
+
+```json
+{
+  "file": "crates/queryd/src/service.rs",
+  "reviewed_at": "2026-04-24T11:06:56Z",
+  "accepted_model": "ollama_cloud/kimi-k2:1t",
+  "accepted_on_attempt": 1,
+  "attempts_made": 1,
+  "tree_split_fired": true,
+  "suggestions_preview": "<truncated-2000-char>",
+  "confidences_per_finding": [92, 90, 88, 85, 75],
+  "confidence_avg": 86,
+  "confidence_min": 75,
+  "findings_count": 5,
+  "gradient_tier": "dry_run",          // auto ≥90 / dry_run ≥70 / simulation ≥50 / block <50
+  "gradient_tier_avg": "dry_run",
+  "alignment_score": 3,                // 1-10 self-rated
+  "output_format": "forensic_json",
+  "verdict": "fail",                   // pass | needs_patch | fail
+  "critical_failures_count": 3,
+  "pseudocode_flags_count": 0,
+  "prd_mismatches_count": 4,
+  "missing_components_count": 6,
+  "verified_components_count": 2,
+  "risk_points_count": 3,
+  "schema_version": 4,
+  "scrum_master_reviewed": true,
+  // ADR-021 fields on pathway trace (NOT this row, see pathway_memory state.json)
+  "pathway_hot_swap_hit": false,
+  "pathway_id": null,
+  "pathway_similarity": null,
+  "pathway_success_rate": null,
+  "rungs_saved": 0
+}
+```
+
+## 5. Applier hardened gates (landed 5e8d87b)
+
+`tests/real-world/scrum_applier.ts` has **5 gates** between emitter output and commit. A patch must pass ALL:
+
+1. **Confidence gate** — emitter's self-reported `confidence >= MIN_CONF` (default 90; 85 with relaxed env). Rejected patches log reason `confidence NN < MM`.
+2. **Size gate** — max 6 lines changed per patch. Prevents cascading cross-file refactors.
+3. **Cargo-green gate** — `cargo check --workspace` must pass. Red build → `git checkout -- file`.
+4. **Warning-count gate** — workspace baseline warning count measured at start; after patch, new count must be `≤ baseline`. Catches unused-import additions and dead_code-after-allow-removed patterns. **THIS GATE CAUGHT 96b46cd's HashSet noise.**
+5. **Rationale-diff token alignment** — rationale text must share at least one non-stopword token with the patch's new_string. Catches "Add destructive SQL filter" claims on `use tracing;` diffs.
+
+Plus:
+- **Deny-list**: `config/`, `ops/`, `auditor/`, `docs/`, `data/`, `mcp-server/`, `ui/`, `sidecar/`, `scripts/` can't be auto-applied (human review required).
+- **Branch guard**: refuses to run on `main`.
+- **Dry-run workspace revert**: in `COMMIT=0` mode, file is reverted after check regardless of outcome — no state pollution between runs.
+
+Every decision logs to `data/_kb/auto_apply.jsonl` (action: `committed` / `build_red_reverted` / `warnings_increased_reverted` / `rationale_mismatch_reverted` / `all_rejected` / `no_patches` / `dry_run_would_commit`).
+
+## 6. Pathway memory (ADR-021)
+
+**Full spec: `docs/DECISIONS.md` ADR-021. Code: `crates/vectord/src/pathway_memory.rs`.**
+
+Three-layer matrix index for compounding semantic-correctness signal:
+
+### Fingerprint (narrow)
+`pathway_id = SHA256(task_class + "|" + file_prefix + "|" + signal_class)` — first 2 path segments (`crates/queryd`) so related files in the same crate share pathways.
+
+### Embedding (similarity vector)
+32-bucket L2-normalized token hash. Tokens include: task_class, file_path, signal_class, per-attempt model+rung+accepted flag, KB chunk source_docs, observer class, bridge libraries, sub-pipeline calls, **semantic_flags**, and **bug_fingerprints (flag+pattern_key)**.
+
+**TS and Rust implementations byte-match** — verified by smoke test showing cosine=1.0 on same input tokens. This is load-bearing for the TS-written traces to be searchable against the Rust-indexed space.
+
+### Hot-swap gate (5-factor AND)
+```
+narrow_fingerprint_matches
+AND audit_consensus.pass != false (null OK during bootstrap)
+AND replay_count >= 3 (probation)
+AND success_rate >= 0.80
+AND NOT retired
+AND similarity(query_vec, stored.pathway_vec) >= 0.90
+```
+
+Replay bookkeeping: on hot-swap, `replay_count++`; if the recommended model succeeded, `replays_succeeded++`; if `replay_count >= 3 AND success_rate < 0.80` → `retired = true` (sticky — prevents oscillation on noise).
+
+### Semantic-correctness layer (ADR-021)
+Each `PathwayTrace` carries:
+- `semantic_flags: Vec<SemanticFlag>` — one of 9 variants: `UnitMismatch`, `TypeConfusion`, `NullableConfusion`, `OffByOne`, `StaleReference`, `PseudoImpl`, `DeadCode`, `WarningNoise`, `BoundaryViolation`
+- `bug_fingerprints: Vec<BugFingerprint>` — `{flag, pattern_key, example, occurrences}` where `pattern_key = "{Flag}:{sorted-top-3-identifiers-joined-by-hyphen}"`. Stable across prose variation.
+- `type_hints_used: Vec<TypeHint>` — `{source, symbol, type_repr}`. Phase E (not yet populated).
+
+**Pre-review enrichment**: scrum calls `POST /vectors/pathway/bug_fingerprints` with `{task_class, file_path, signal_class, limit}` — returns aggregated fingerprints sorted by occurrences descending. If any, a `📚 PATHWAY MEMORY` preamble is prepended to the reviewer prompt with "this file area had these patterns before — check for recurrences."
+
+**Post-review extractor** (Phase D, `scrum_master_pipeline.ts`): walks reviewer markdown line-by-line, finds lines containing a `SemanticFlag` variant, extracts identifier-shaped backtick-quoted tokens, filters out flag names + Rust keywords (self/mut/async/etc), sorts and takes top 3, builds `pattern_key = "{Flag}:{tokens}"`.
+
+### HTTP surface (on gateway port 3100)
+| Endpoint | Purpose |
+|---|---|
+| `POST /vectors/pathway/insert` | write a full PathwayTrace |
+| `POST /vectors/pathway/query` | hot-swap candidate check (returns `{candidate: null}` or `{candidate: {...}}`) |
+| `POST /vectors/pathway/record_replay` | update replay_count + success_rate after hot-swap |
+| `GET /vectors/pathway/stats` | totals + reuse_rate + replay_success_rate |
+| `POST /vectors/pathway/bug_fingerprints` | aggregated fingerprints by narrow fingerprint (for pre-review preamble) |
+
+### State persistence
+`data/_pathway_memory/state.json` — JSON dump of all buckets. Loaded at gateway boot (`crates/gateway/src/main.rs` has `pwm.load_from_storage().await`).
+
+## 7. Current state (2026-04-24 end of session)
+
+### Commits on branch `scrum/auto-apply-19814` since iter-5 baseline
+
+| # | SHA | Subject |
+|---|---|---|
+| 1 | `25ea3de` | observer fix — route LLM Team escalation to `/v1/chat` qwen3-coder |
+| 2 | `8b77d67` | OpenRouter rescue ladder + tree-split reducer + first auto-applier |
+| 3 | `96b46cd` | first auto-applied commit (later found misleading) |
+| 4 | `5e8d87b` | cleanup + applier hardening (warning + rationale + dry-run gates) |
+| 5 | `9cc0ceb` | P42-002 — truth gate into queryd `/sql` + `/paged` paths |
+| 6 | `2f8b347` | pathway_memory base (PathwayTrace, hot-swap, 18 tests) |
+| 7 | `86901f8` | queryd/delta.rs 6-line unit-mismatch fix |
+| 8 | `92df0e9` | ADR-021 spec |
+| 9 | `0a0843b` | ADR-021 Phases A+B+C (semantic_flags, prompt tags, preamble endpoint) |
+| 10 | `ee31424` | ADR-021 Phase D (fingerprint extractor) |
+| 11 | `f4cff66` | Phase D fix — strip flag names + Rust keywords from pattern_keys |
+
+### Matrix index state
+- **12 pathway traces** in `data/_pathway_memory/state.json`
+- **11 distinct bug fingerprints** across 4 Flag categories on `crates/queryd` narrow fingerprint (1 manually seeded + 10 extracted)
+- **0 hot-swaps fired** (probation requires ≥3 replays per pathway; none reached yet)
+
+### Active in-flight
+- Iter 9 complete; next iter 10+ will use cleaner fingerprint extractor (`f4cff66`)
+- 4 "noisy" pattern_keys from iter-9-file-1 pre-fix run (e.g., `DeadCode:DeadCode`) — dormant, won't match future output, acceptable dead entries
+
+### Queued (not yet implemented)
+- **Phase E** — `type_hints_used` population from `catalogd` column types, Arrow `RecordBatch.schema()`, Rust struct field types. Feeds typed context to reviewer prompt.
+- **Auditor → pathway audit_consensus wire** — activates the strict-audit gate (currently lenient: null bootstraps, only explicit `false` blocks).
+- **VCP UI cards** for "top bug fingerprints in last N iters" + "new patterns learned this iter"
+
+## 8. How to run a new iteration
+
+```bash
+# Default 3 files (playbook_memory.rs, doc_drift.rs, auditor/audit.ts)
+LH_SCRUM_FORENSIC=/home/profit/lakehouse/docs/SCRUM_FORENSIC_PROMPT.md \
+LH_SCRUM_PROPOSAL=/home/profit/lakehouse/docs/SCRUM_FIX_WAVE.md \
+bun run tests/real-world/scrum_master_pipeline.ts
+
+# Targeted files:
+LH_SCRUM_FILES="/home/profit/lakehouse/crates/queryd/src/delta.rs,/home/profit/lakehouse/crates/queryd/src/service.rs" \
+LH_SCRUM_FORENSIC=... LH_SCRUM_PROPOSAL=... \
+bun run tests/real-world/scrum_master_pipeline.ts
+
+# Dry-run auto-applier against the latest scrum output:
+LH_APPLIER_MIN_CONF=85 LH_APPLIER_MAX_FILES=10 \
+LH_APPLIER_MODEL=qwen3-coder:480b \
+LH_APPLIER_BRANCH=scrum/auto-apply-19814 \
+bun run tests/real-world/scrum_applier.ts
+
+# Actually commit (ONLY after dry-run looks clean):
+LH_APPLIER_COMMIT=1 LH_APPLIER_MIN_CONF=85 LH_APPLIER_MAX_FILES=10 \
+LH_APPLIER_MODEL=qwen3-coder:480b \
+LH_APPLIER_BRANCH=scrum/auto-apply-19814 \
+bun run tests/real-world/scrum_applier.ts
+```
+
+## 9. Verify services before running
+
+```bash
+# Gateway (port 3100) — must be up; pathway endpoints are here
+curl -s http://localhost:3100/health            # "lakehouse ok"
+curl -s http://localhost:3100/vectors/pathway/stats   # pathway memory totals
+
+# UI (port 3950) — VCP dashboard + /data/pathway_stats aggregation
+curl -s http://localhost:3950/data/pathway_stats
+
+# Observer (port 3800) — event receiver + LLM Team escalation
+curl -s http://localhost:3800/health 2>/dev/null || true
+
+# Sidecar (port 3200) — Python embed
+curl -s http://localhost:3200/health 2>/dev/null || true
+
+# LLM Team (port 5000) — /api/run?mode=extract ONLY registered mode
+# (others like code_review/patch/refactor return "Unknown mode")
+curl -s http://localhost:5000/health 2>/dev/null || true
+```
+
+If gateway missing new routes after code change: `cargo build --release -p gateway && sudo systemctl restart lakehouse.service`.
+
+If UI missing new routes: kill old `bun run ui/server.ts` and restart (not a systemd service right now).
+
+## 10. Where things live (code pointers)
+
+| Concern | File |
+|---|---|
+| Scrum orchestrator | `tests/real-world/scrum_master_pipeline.ts` |
+| Scrum ladder constant | same file, `const LADDER` line ~92 |
+| Tree-split reducer | same file, `async function treeSplitFile` |
+| Forensic prompt preamble (loaded via env) | `docs/SCRUM_FORENSIC_PROMPT.md` |
+| Fix-wave proposal preamble | `docs/SCRUM_FIX_WAVE.md` |
+| Scrum iter notes | `docs/SCRUM_LOOP_NOTES.md` |
+| Auto-applier | `tests/real-world/scrum_applier.ts` |
+| Applier audit trail | `data/_kb/auto_apply.jsonl` |
+| Scrum reviews KB | `data/_kb/scrum_reviews.jsonl` |
+| Model trust journal | `data/_kb/model_trust.jsonl` |
+| Pathway memory module | `crates/vectord/src/pathway_memory.rs` |
+| Pathway HTTP handlers | `crates/vectord/src/service.rs` (bottom) |
+| Pathway state on disk | `data/_pathway_memory/state.json` |
+| VCP UI server | `ui/server.ts` |
+| VCP UI client | `ui/ui.js` + `ui/ui.css` + `ui/index.html` |
+| Observer | `mcp-server/observer.ts` |
+| Auditor | `auditor/audit.ts` |
+| LLM Team extract client | `auditor/fact_extractor.ts` |
+| ADR-021 spec | `docs/DECISIONS.md` ADR-021 |
+
+## 11. Key memory files a fresh session should read
+
+From `/root/.claude/projects/-home-profit/memory/`:
+
+- `project_scrum_pipeline.md` — updated state of the scrum iterations
+- `project_first_auto_apply.md` — 96b46cd story + cleanup + hardening evidence from iter 7
+- `feedback_semantic_correctness_via_matrix.md` — J's insight on compounding, the ADR-021 rule
+- `feedback_endpoint_probe_discipline.md` — GET 405 is not endpoint validation
+- `reference_llm_team_modes.md` — only `extract` is registered on port 5000
+- `feedback_scrum_cloud_first.md` — scrum/audit/enrich pipelines use cloud first
+- `feedback_cloud_determinism.md` — cloud N=3 consensus + qwen3-coder tie-breaker
+
+## 12. Known gotchas
+
+- **Gateway restart needed after Rust route additions.** `sudo systemctl restart lakehouse.service` — the service is systemd-managed.
+- **UI server needs manual restart** after `ui/server.ts` changes (no systemd unit). Kill old `bun` pid, restart with `bun run ui/server.ts &`.
+- **LLM Team mode `code_review` doesn't exist** — only `extract` is registered in `/root/llm_team_ui.py`. Don't wire new features to "Unknown mode" endpoints. See `reference_llm_team_modes.md`.
+- **OpenRouter free-tier 429s during consensus probes** are normal (rate-limited upstream). In the production ladder they hit as last-resort rescue with seconds-to-minutes gap; different traffic pattern than rapid-fire consensus runs.
+- **Openrouter minimax-m2.5:free has a 45s timeout** — not in ladder, only for one-off probes.
+- **Probation period is 3 replays** before hot-swap can fire. On a fresh install, no hot-swap fires until a pathway has been re-visited ≥3 times.

mcp-server/index.ts

@@ -18,6 +18,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import { SSEServerTransport } from "@modelcontextprotocol/sdk/server/sse.js";
 import { z } from "zod";
 import { startTrace, logSpan, logGeneration, scoreTrace, flush as flushTraces } from "./tracing.js";
+import { buildPermitBrief } from "./entity.js";
 
 const BASE = process.env.LAKEHOUSE_URL || "http://localhost:3100";
 const PORT = parseInt(process.env.MCP_PORT || "3700");
@@ -39,20 +40,10 @@ async function api(method: string, path: string, body?: any, retries = 2) {
       const ms = Date.now() - t0;
       let parsed: any;
       try { parsed = JSON.parse(text); } catch { parsed = { raw: text, status: resp.status }; }
-      return parsed;
-    } catch (e: any) {
-      if (attempt === retries) throw e;
-      if (e.message?.includes("socket connection was closed") || e.message?.includes("ECONNREFUSED")) {
-        await Bun.sleep(500 * (attempt + 1));
-        continue;
-      }
-      throw e;
-    }
-  }
-  throw new Error("unreachable");
-}
 
-  // Trace the call if we have an active trace
+      // Trace the call if we have an active trace. Pre-existing edit had
+      // this block at module scope, dangling after the closing brace of
+      // api() — parsed broken until fixed 2026-04-24.
       if (activeTrace) {
         const isGen = path.includes("/generate");
         if (isGen) {
@@ -73,6 +64,16 @@ async function api(method: string, path: string, body?: any, retries = 2) {
       }
 
       return parsed;
+    } catch (e: any) {
+      if (attempt === retries) throw e;
+      if (e.message?.includes("socket connection was closed") || e.message?.includes("ECONNREFUSED")) {
+        await Bun.sleep(500 * (attempt + 1));
+        continue;
+      }
+      throw e;
+    }
+  }
+  throw new Error("unreachable");
 }
 
 const server = new McpServer({ name: "lakehouse", version: "1.0.0" });
@@ -960,6 +961,61 @@ async function main() {
           return new Response(Bun.file(import.meta.dir + "/console.html"));
         }
 
+        // ─── Contractor / entity drill-down page ───
+        // Single-contractor portfolio view across every wired source:
+        // OSHA national, Chicago history, ticker chart, parent link,
+        // federal contracts, debarment, unions, training. Click any
+        // contractor name in a permit Entity Brief to land here.
+        if (url.pathname === "/contractor") {
+          return new Response(Bun.file(import.meta.dir + "/contractor.html"), {
+            headers: { ...cors, "Content-Type": "text/html" },
+          });
+        }
+        if (url.pathname === "/intelligence/contractor_profile" && req.method === "POST") {
+          const start = Date.now();
+          try {
+            const b = (await req.json().catch(() => ({}))) as { name?: string };
+            if (!b.name) return err("missing name", 400);
+            // Use the entity-brief library directly — single entity, all sources.
+            const { fetchOshaBrief, fetchTickerBrief, fetchContractorHistory, fetchParentLink, fetchFederalContracts, fetchDebarmentBrief, fetchNlrbBriefReal, fetchIlsosBrief, fetchNewsMentions, fetchDiversityCerts, scoreNewsSentiment, fetchBlsConstructionTrend, normalizeEntityName, entityTicker } = await import("./entity.js");
+            const [osha, stock, history, parent_link, federal, debarment, nlrb, ilsos, news, diversity, macro] = await Promise.all([
+              fetchOshaBrief(b.name),
+              fetchTickerBrief(b.name),
+              fetchContractorHistory(b.name),
+              fetchParentLink(b.name),
+              fetchFederalContracts(b.name),
+              fetchDebarmentBrief(b.name),
+              fetchNlrbBriefReal(b.name),
+              fetchIlsosBrief(b.name),
+              fetchNewsMentions(b.name),
+              fetchDiversityCerts(b.name),
+              fetchBlsConstructionTrend(),
+            ]);
+            const news_sentiment = news ? scoreNewsSentiment(news) : null;
+            return ok({
+              key: normalizeEntityName(b.name),
+              display_name: b.name,
+              ticker: entityTicker(b.name),
+              osha,
+              stock,
+              history,
+              parent_link,
+              federal,
+              debarment,
+              nlrb,
+              ilsos,
+              news,
+              news_sentiment,
+              diversity,
+              macro,
+              generated_at: new Date().toISOString(),
+              duration_ms: Date.now() - start,
+            });
+          } catch (e: any) {
+            return err(`contractor_profile: ${e.message}`, 500);
+          }
+        }
+
         // Intelligence: Market data — public building permits → staffing demand forecast
         if (url.pathname === "/intelligence/market" && req.method === "POST") {
           const start = Date.now();
@@ -1158,14 +1214,86 @@ async function main() {
         // a PROPOSED fill drawn from our 500K worker bench. Surfaces the
         // meta-index dimension directly: "what past similar fills had in
         // common" for this role + geo.
+        // Architecture signals — the "our substrate is better than the
+        // alternatives" proof surface. Pulls live health numbers so the
+        // dashboard can show, per-card or in a top bar, that the claims
+        // we make in the PRD (instant searches, self-regulation,
+        // hot-swap, indexed-at-ingest) are verifiable right now.
+        if (url.pathname === "/intelligence/arch_signals" && (req.method === "GET" || req.method === "POST")) {
+          try {
+            const t0 = Date.now();
+            // Index freshness + shape (hot-swap + clever-index claims)
+            const idxRaw = await fetch("http://localhost:3100/vectors/indexes/workers_500k_v1", {
+              signal: AbortSignal.timeout(3000),
+            }).then(r => r.ok ? r.json() : null).catch(() => null);
+
+            // Playbook memory — "self-regulates via learned playbooks"
+            const pbmRaw = await fetch("http://localhost:3100/vectors/playbook_memory/stats", {
+              signal: AbortSignal.timeout(3000),
+            }).then(r => r.ok ? r.json() : null).catch(() => null);
+
+            // Pathway memory — ADR-021 compounding-bug-grammar surface
+            const pwmRaw = await fetch("http://localhost:3100/vectors/pathway/stats", {
+              signal: AbortSignal.timeout(3000),
+            }).then(r => r.ok ? r.json() : null).catch(() => null);
+
+            // Live instant-search probe — one trivial hybrid call so the
+            // latency number on screen is fresh, not cached.
+            const probeT0 = Date.now();
+            await api("POST", "/vectors/hybrid", {
+              index_name: "workers_500k_v1",
+              filter_dataset: "workers_500k",
+              id_column: "worker_id",
+              sql_filter: "state = 'OH'",
+              question: "production worker",
+              top_k: 3, generate: false,
+            }).catch(() => ({}));
+            const probeMs = Date.now() - probeT0;
+
+            return ok({
+              generated_at: new Date().toISOString(),
+              duration_ms: Date.now() - t0,
+              index: idxRaw ? {
+                name: idxRaw.index_name,
+                source: idxRaw.source,
+                model: idxRaw.model_name,
+                dimensions: idxRaw.dimensions,
+                chunk_count: idxRaw.chunk_count,
+                doc_count: idxRaw.doc_count,
+                created_at: idxRaw.created_at,
+                backend: idxRaw.vector_backend,
+                last_used: idxRaw.last_used ?? null,
+                build_signature: idxRaw.build_signature ?? null,
+              } : null,
+              playbook_memory: pbmRaw ? {
+                entries: pbmRaw.entries_count ?? pbmRaw.count ?? 0,
+                rebuilt_at: pbmRaw.last_rebuilt_at ?? null,
+              } : null,
+              pathway_memory: pwmRaw ? {
+                total_pathways: pwmRaw.total_pathways ?? 0,
+                retired: pwmRaw.retired ?? 0,
+                with_audit_pass: pwmRaw.with_audit_pass ?? 0,
+                total_replays: pwmRaw.total_replays ?? 0,
+              } : null,
+              instant_search_probe_ms: probeMs,
+            });
+          } catch (e: any) {
+            return err(`arch_signals: ${e.message}`, 500);
+          }
+        }
+
         if (url.pathname === "/intelligence/permit_contracts" && req.method === "POST") {
           const start = Date.now();
           try {
             const permitUrl = "https://data.cityofchicago.org/resource/ydr8-5enu.json";
             // Recent + substantial permits only — skip tiny ones that
             // don't imply real staffing demand.
+            // Include contact_1 + contact_2 fields so the Entity Brief
+            // panel on each card can populate without a second fetch.
+            // Contacts identify the applicant / contractor by name —
+            // those are the keys we pass to OSHA/ILSOS enrichment.
             const permits: any[] = await fetch(
-              `${permitUrl}?$select=permit_type,work_type,work_description,reported_cost,street_number,street_direction,street_name,community_area,issue_date&`
+              `${permitUrl}?$select=id,permit_type,work_type,work_description,reported_cost,street_number,street_direction,street_name,community_area,issue_date,contact_1_name,contact_1_type,contact_2_name,contact_2_type&`
               + `$where=reported_cost>250000 AND issue_date>'2025-06-01'`
               + `&$order=issue_date DESC&$limit=6`
             ).then(r => r.json()).catch(() => []);
@@ -1193,6 +1321,11 @@ async function main() {
               // query path exactly. k=200 to ensure boost fires across
               // the full memory surface (the embedding-discrimination
               // narrowness means under-k silently misses endorsements).
+              //
+              // Timed so the UI can surface "instant search from clever
+              // indexing at ingest" — the architecture claim J wants
+              // visible. Each contract card shows its hybrid latency.
+              const hybridT0 = Date.now();
               const searchRes = await api("POST", "/vectors/hybrid", {
                 index_name: "workers_500k_v1",
                 filter_dataset: "workers_500k",
@@ -1202,6 +1335,7 @@ async function main() {
                 top_k: 5, generate: false,
                 use_playbook_memory: true, playbook_memory_k: 200,
               }).catch(() => ({ sources: [] as any[] }));
+              const hybridMs = Date.now() - hybridT0;
 
               // Path 2 — discovered patterns for this role in this city.
               const patternRes = await api("POST", "/vectors/playbook_memory/patterns", {
@@ -1240,14 +1374,72 @@ async function main() {
               else if (daysToDeadline <= 21) urgency = "soon";
               else urgency = "scheduled";
 
+              // Fill-probability ramp — staffing-industry heuristic.
+              // Base probability by pool_size (how many available workers
+              // match the role+geo), decayed by days-remaining. Produces
+              // a curve the UI can sparkline.
+              const poolSize = (searchRes.sql_matches ?? 0) as number;
+              const basePFill = poolSize >= count * 20 ? 0.95
+                              : poolSize >= count * 10 ? 0.85
+                              : poolSize >= count * 5  ? 0.70
+                              : poolSize >= count * 2  ? 0.55
+                              : poolSize >= count      ? 0.35
+                              : 0.15;
+              const fillByDay = [0, 3, 7, 14, 21, 30].map((d) => {
+                // Front-loaded: most fills land in first 7 days; tail
+                // falls off quickly. This is a Weibull-ish shape that
+                // matches real staffing data we've seen.
+                const ramp = d === 0 ? 0.0
+                           : d <= 3  ? 0.35
+                           : d <= 7  ? 0.65
+                           : d <= 14 ? 0.85
+                           : d <= 21 ? 0.95
+                           : 1.0;
+                return { day: d, cumulative_pct: Math.round(basePFill * ramp * 100) };
+              });
+
+              // Economics — "as though the contracts were accepted and
+              // filled." 40 hrs/week, default 12-week contract. Margin
+              // = (bill - avg_pay) × count × hours. Payout window is
+              // fill_date + 30d billing cycle.
+              const weeksAssumed = 12;
+              const hoursPerWeek = 40;
+              const avgPayRate = sources.length
+                ? sources.reduce((s, c) => s + (c.implied_pay_rate || 0), 0) / sources.length
+                : contractBillRate / BILL_MARKUP;
+              const grossRevenue = contractBillRate * count * hoursPerWeek * weeksAssumed;
+              const grossMargin = (contractBillRate - avgPayRate) * count * hoursPerWeek * weeksAssumed;
+              const overBillCount = sources.filter((c) => c.over_bill_rate).length;
+              const overBillPoolMargin = sources
+                .filter((c) => c.over_bill_rate)
+                .reduce((s, c) => s + (c.implied_pay_rate - contractBillRate) * hoursPerWeek * weeksAssumed, 0);
+
+              // Shift inference from permit work_type + description.
+              // Construction defaults to 1st-shift (day). Heavy civil or
+              // facility work sometimes runs 2nd or split-shift. 3rd
+              // (overnight) is rare in commercial construction but real
+              // for maintenance / emergency calls.
+              const descLower = ((p.work_description || "") + " " + (p.work_type || "")).toLowerCase();
+              const shifts: string[] = ["1st"]; // default day
+              if (/night|overnight|24\s*hr|emergency/.test(descLower)) shifts.push("3rd");
+              if (/multi.?shift|round.?the.?clock|double.?shift/.test(descLower)) shifts.push("2nd");
+              if (/weekend|saturday|sunday/.test(descLower)) shifts.push("4th");
+
               contracts.push({
                 permit: {
+                  id: p.id,
                   cost,
                   work_type: p.work_type || "General construction",
                   description: (p.work_description || "").substring(0, 140),
                   address: `${p.street_number || ""} ${p.street_direction || ""} ${p.street_name || ""}`.trim(),
                   community_area: p.community_area,
                   issue_date: (p.issue_date || "").substring(0, 10),
+                  // Contacts — used by /intelligence/permit_entities to
+                  // enrich each card with OSHA + ILSOS on expand.
+                  contact_1_name: p.contact_1_name || "",
+                  contact_1_type: p.contact_1_type || "",
+                  contact_2_name: p.contact_2_name || "",
+                  contact_2_type: p.contact_2_type || "",
                 },
                 implied_bill_rate: contractBillRate,
                 timeline: {
@@ -1260,12 +1452,32 @@ async function main() {
                   role,
                   count,
                   city, state,
-                  pool_size: searchRes.sql_matches,
+                  pool_size: poolSize,
                   candidates: sources,
                 },
                 discovered_pattern: patternRes.discovered_pattern,
                 pattern_matched: patternRes.matched_playbooks ?? 0,
                 pattern_workers_examined: patternRes.total_workers_examined ?? 0,
+                // ADR-021 / PRD architecture claims surface — these fields
+                // let the UI show "instant search from clever indexing"
+                // and the fill economics beyond bill rate alone.
+                search_latency_ms: hybridMs,
+                fill_probability: {
+                  base_pct: Math.round(basePFill * 100),
+                  curve: fillByDay,
+                },
+                economics: {
+                  avg_pay_rate: Math.round(avgPayRate * 100) / 100,
+                  hours_per_week: hoursPerWeek,
+                  weeks_assumed: weeksAssumed,
+                  gross_revenue: Math.round(grossRevenue),
+                  gross_margin: Math.round(grossMargin),
+                  margin_pct: grossRevenue > 0 ? Math.round((grossMargin / grossRevenue) * 100) : 0,
+                  payout_window_days: [30, 45],
+                  over_bill_count: overBillCount,
+                  over_bill_pool_margin_at_risk: Math.round(overBillPoolMargin),
+                },
+                shifts_needed: shifts,
               });
             }
 
@@ -1281,6 +1493,58 @@ async function main() {
           }
         }
 
+        // Intelligence: per-permit entity brief — OSHA + ILSOS + property
+        // Takes a permit identifier (we look it up from Chicago Socrata) or
+        // raw contact fields directly from the client. Returns an "ETF
+        // basket" shape: property + entities + per-entity risk factors.
+        // OSHA is live-scraped (cached 30d). ILSOS returns a structured
+        // placeholder because apps.ilsos.gov blocks our ASN.
+        if (url.pathname === "/intelligence/permit_entities" && req.method === "POST") {
+          const start = Date.now();
+          try {
+            const b = await req.json().catch(() => ({})) as {
+              permit_id?: string;
+              address?: string;
+              work_type?: string;
+              contact_1_name?: string;
+              contact_1_type?: string;
+              contact_2_name?: string;
+              contact_2_type?: string;
+              fetch_osha?: boolean;
+              fetch_ilsos?: boolean;
+            };
+            // If the caller didn't pass contact fields but did pass a
+            // permit_id, go pull the record from Chicago Socrata.
+            let permit = b;
+            if (b.permit_id && !b.contact_1_name) {
+              const u = `https://data.cityofchicago.org/resource/ydr8-5enu.json?$where=id='${encodeURIComponent(b.permit_id)}'`;
+              const rows = (await fetch(u).then((r) => r.json())) as any[];
+              const p = rows?.[0];
+              if (p) {
+                const addr = [p.street_number, p.street_direction, p.street_name]
+                  .filter(Boolean)
+                  .join(" ");
+                permit = {
+                  permit_id: b.permit_id,
+                  address: addr,
+                  work_type: p.work_type,
+                  contact_1_name: p.contact_1_name,
+                  contact_1_type: p.contact_1_type,
+                  contact_2_name: p.contact_2_name,
+                  contact_2_type: p.contact_2_type,
+                };
+              }
+            }
+            const brief = await buildPermitBrief(permit, {
+              fetchOsha: b.fetch_osha !== false,
+              fetchIlsos: b.fetch_ilsos !== false,
+            });
+            return ok({ ...brief, duration_ms: Date.now() - start });
+          } catch (e: any) {
+            return err(`permit_entities: ${e.message}`, 500);
+          }
+        }
+
         // Removed 2026-04-20: /intelligence/learn was a legacy CSV writer
         // that destructively re-wrote successful_playbooks. /log and
         // /log_failure replace it cleanly via /vectors/playbook_memory/seed

mcp-server/observer.ts

@@ -141,6 +141,270 @@ async function persistOp(op: ObservedOp) {
 }
 
 
+// ─── LLM Team escalation (code_review mode) ───
+//
+// When recent failures on a single sig_hash cross a threshold the
+// local qwen2.5 analysis is probably insufficient. J's 2026-04-24
+// direction: "the observer would trigger to give more context" —
+// route failure clusters to LLM Team's specialized code_review mode
+// (via /api/run) so richer structured signal lands in the KB for
+// scrum + auditor + playbook memory to consume next pass.
+//
+// Non-destructive: runs in parallel to the existing qwen2.5 analysis,
+// never replaces it. Writes to data/_kb/observer_escalations.jsonl
+// as a dedicated audit surface.
+
+const LLM_TEAM = process.env.LH_LLM_TEAM_URL ?? "http://localhost:5000";
+const LLM_TEAM_ESCALATIONS = "/home/profit/lakehouse/data/_kb/observer_escalations.jsonl";
+const ESCALATION_THRESHOLD = 3; // N+ failures on same sig_hash triggers
+
+async function escalateFailureClusterToLLMTeam(sigHash: string, cluster: ObservedOp[]) {
+  // Package the failure cluster as a single context blob. Originally
+  // I routed this to LLM Team's `code_review` mode at /api/run, but
+  // that mode isn't registered in llm_team_ui.py — it returned
+  // "Unknown mode" on every call. Revised 2026-04-24: route directly
+  // to the gateway's /v1/chat with provider=ollama_cloud + qwen3-coder:480b
+  // (the coding specialist that's rung 2 of the scrum ladder, proven
+  // to produce substantive structured reviews). Fire-and-forget so
+  // downstream failures don't block observer's normal loop.
+  const context = cluster.slice(-8).map((o, i) =>
+    `[${i + 1}] endpoint=${o.endpoint} input=${o.input_summary} error=${o.error ?? "?"}`
+  ).join("\n");
+  const prompt = `sig_hash=${sigHash} · ${cluster.length} failures on the same signature:\n\n${context}\n\nReview this failure cluster. Identify:\n1. Likely root cause (single sentence).\n2. Files most likely responsible (path hints).\n3. Concrete fix direction (under 3 sentences).\n4. Confidence: NN%\n\nBe specific, not generic.`;
+
+  try {
+    const resp = await fetch(`${LAKEHOUSE}/v1/chat`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        provider: "ollama_cloud",
+        model: "qwen3-coder:480b",
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: 800,
+        temperature: 0.2,
+      }),
+      signal: AbortSignal.timeout(60000),
+    });
+    if (!resp.ok) {
+      console.error(`[observer] escalation /v1/chat ${resp.status}: ${(await resp.text()).slice(0, 200)}`);
+      return;
+    }
+    const j: any = await resp.json();
+    const analysis = j?.choices?.[0]?.message?.content ?? "";
+
+    // Audit row stays schema-compatible with the prior implementation —
+    // downstream consumers see structured fields regardless of the
+    // review-source change. Facts/entities stay empty (this call is
+    // direct-model, not extract-mode); the raw analysis carries the
+    // signal.
+    const row = {
+      ts: new Date().toISOString(),
+      source: "observer_escalation",
+      mode: "direct_chat_qwen3_coder_480b",
+      sig_hash: sigHash,
+      cluster_size: cluster.length,
+      cluster_staffer: cluster[0]?.staffer_id,
+      cluster_endpoint: cluster[0]?.endpoint,
+      prompt_tokens: j?.usage?.prompt_tokens ?? 0,
+      completion_tokens: j?.usage?.completion_tokens ?? 0,
+      analysis: analysis.slice(0, 4000),
+    };
+    const { appendFile } = await import("node:fs/promises");
+    await appendFile(LLM_TEAM_ESCALATIONS, JSON.stringify(row) + "\n");
+    console.error(
+      `[observer] escalated sig_hash=${sigHash.slice(0, 8)} · cluster=${cluster.length} · ${analysis.length} chars`
+    );
+  } catch (e) {
+    console.error(`[observer] escalation failed: ${(e as Error).message}`);
+  }
+}
+
+// Track which sig_hashes we've already escalated this session so we
+// don't hammer LLM Team on every analyzeErrors tick when a cluster
+// persists across cycles.
+const escalatedSigHashes = new Set<string>();
+
+// ─── Hand-review for scrum/agent candidate responses (2026-04-25) ───
+//
+// Observer is OUTSIDE the scrum loop's epistemic scope, so its verdict
+// can be treated as truth about whether a candidate review is grounded.
+// Two-tier evaluator:
+//   1. Try cloud LLM (qwen3-coder:480b) — semantic judgment with
+//      response + source excerpt + grounding stats as context.
+//   2. On cloud failure (throttle/timeout) → deterministic heuristic
+//      over grounding_pct + total_quotes. Marked source: "heuristic"
+//      so consumers can tell which rung produced the verdict.
+// Every verdict is persisted to data/_kb/observer_reviews.jsonl.
+
+const OBSERVER_REVIEWS = "/home/profit/lakehouse/data/_kb/observer_reviews.jsonl";
+
+interface HandReviewInput {
+  file_path: string;
+  model: string;
+  response: string;
+  source_content: string;
+  grounding_stats: { total: number; grounded: number; groundedPct: number | null };
+  attempt: number;
+}
+
+interface HandReviewVerdict {
+  verdict: "accept" | "reject" | "cycle";
+  confidence: number;
+  notes: string;
+  source: "cloud" | "heuristic";
+}
+
+async function handReview(input: HandReviewInput): Promise<HandReviewVerdict> {
+  const t0 = Date.now();
+  let verdict: HandReviewVerdict;
+
+  try {
+    verdict = await cloudHandReview(input);
+  } catch (e) {
+    console.error(`[observer/review] cloud failed (${(e as Error).message}); using heuristic`);
+    verdict = heuristicHandReview(input);
+  }
+
+  // Persist regardless of source so we can later compare cloud vs
+  // heuristic verdicts on the same input and tune the heuristic.
+  const row = {
+    ts: new Date().toISOString(),
+    file_path: input.file_path,
+    model: input.model,
+    attempt: input.attempt,
+    response_chars: input.response.length,
+    grounding_stats: input.grounding_stats,
+    verdict: verdict.verdict,
+    confidence: verdict.confidence,
+    notes: verdict.notes,
+    source: verdict.source,
+    duration_ms: Date.now() - t0,
+  };
+  try {
+    const { appendFile } = await import("node:fs/promises");
+    await appendFile(OBSERVER_REVIEWS, JSON.stringify(row) + "\n");
+  } catch { /* best-effort persistence */ }
+
+  return verdict;
+}
+
+async function cloudHandReview(input: HandReviewInput): Promise<HandReviewVerdict> {
+  const grounded = input.grounding_stats.grounded;
+  const total = input.grounding_stats.total;
+  const pct = input.grounding_stats.groundedPct;
+  // Truncate to keep the prompt under typical context windows.
+  // 2000 + 4000 = ~6000 chars ≈ 1500 tokens, plus response context.
+  const responseExcerpt = input.response.slice(0, 2000);
+  const sourceExcerpt = input.source_content.slice(0, 4000);
+
+  const prompt = `You are a code-review quality observer. Decide whether the following automated review is grounded in the actual source — not invented, not hallucinated.
+
+FILE: ${input.file_path}
+MODEL: ${input.model}
+ATTEMPT: ${input.attempt}
+ANCHOR GROUNDING: ${grounded}/${total} backtick-quoted snippets matched the source verbatim${pct !== null ? ` (${pct}%)` : ""}
+
+REVIEW (first 2000 chars):
+\`\`\`
+${responseExcerpt}
+\`\`\`
+
+SOURCE EXCERPT (first 4000 chars):
+\`\`\`
+${sourceExcerpt}
+\`\`\`
+
+Respond ONLY with a JSON object:
+{
+  "verdict": "accept" | "reject" | "cycle",
+  "confidence": 0-100,
+  "notes": "<1-2 sentences on what makes this grounded or hallucinated>"
+}
+
+- accept: review references real symbols/lines in source; findings could be acted on.
+- reject: review invents APIs, fabricates calls, contradicts source. Do NOT record.
+- cycle: review is mediocre — partially grounded but wrong shape, try a stronger model.`;
+
+  // Hand-review uses paid OpenRouter so it sidesteps the Ollama Cloud
+  // throttle that drove every prior iter into the heuristic fallback.
+  // Grok 4.1 fast: $0.20 in / $0.50 out per M tokens, 2M ctx. A typical
+  // hand-review (~6K input + 300 output) costs ~$0.0014. Selected via
+  // J directive 2026-04-25 ("best model under $0.72/M").
+  const resp = await fetch(`${LAKEHOUSE}/v1/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      provider: "openrouter",
+      model: "x-ai/grok-4.1-fast",
+      messages: [{ role: "user", content: prompt }],
+      max_tokens: 300,
+      temperature: 0.0,
+    }),
+    signal: AbortSignal.timeout(45000),
+  });
+  if (!resp.ok) {
+    throw new Error(`/v1/chat ${resp.status}: ${(await resp.text()).slice(0, 200)}`);
+  }
+  const j: any = await resp.json();
+  const content = (j?.choices?.[0]?.message?.content ?? "").trim();
+  // Pull JSON object from the response — model may wrap it in prose.
+  const m = content.match(/\{[\s\S]*\}/);
+  if (!m) throw new Error(`no JSON object in response: ${content.slice(0, 100)}`);
+  const parsed = JSON.parse(m[0]);
+  const v = String(parsed.verdict ?? "accept").toLowerCase();
+  return {
+    verdict: (v === "reject" || v === "cycle") ? v as "reject" | "cycle" : "accept",
+    confidence: Number(parsed.confidence ?? 50),
+    notes: String(parsed.notes ?? "").slice(0, 500),
+    source: "cloud",
+  };
+}
+
+function heuristicHandReview(input: HandReviewInput): HandReviewVerdict {
+  // Deterministic fallback when cloud is throttled. Conservative:
+  // only flip to reject when the evidence is overwhelming, otherwise
+  // accept (fall-open principle — observer is policy, not blocker).
+  const total = input.grounding_stats.total;
+  const pct = input.grounding_stats.groundedPct;
+  const respLen = input.response.length;
+
+  // Too short to be a real review
+  if (respLen < 1500) {
+    return { verdict: "reject", confidence: 80, notes: `response too short (${respLen} chars)`, source: "heuristic" };
+  }
+  // Below 5 quotes — not enough signal to judge grounding; accept
+  if (total < 5 || pct === null) {
+    return { verdict: "accept", confidence: 50, notes: `insufficient quote signal (${total} quotes); accepting`, source: "heuristic" };
+  }
+  // Very heavy hallucination
+  if (pct < 20) {
+    return { verdict: "reject", confidence: 85, notes: `low grounding (${pct}% of ${total} quotes)`, source: "heuristic" };
+  }
+  // Mediocre — cycle to a stronger model
+  if (pct < 50) {
+    return { verdict: "cycle", confidence: 65, notes: `mediocre grounding (${pct}% of ${total} quotes); try stronger`, source: "heuristic" };
+  }
+  // Good enough
+  return { verdict: "accept", confidence: 75, notes: `grounding ${pct}% of ${total} quotes`, source: "heuristic" };
+}
+
+async function maybeEscalate(failures: ObservedOp[]) {
+  // Group failures by sig_hash
+  const bySig = new Map<string, ObservedOp[]>();
+  for (const f of failures) {
+    const k = f.sig_hash ?? "__no_sig__";
+    (bySig.get(k) ?? bySig.set(k, []).get(k)!).push(f);
+  }
+  for (const [sigHash, cluster] of bySig) {
+    if (sigHash === "__no_sig__") continue;
+    if (cluster.length < ESCALATION_THRESHOLD) continue;
+    if (escalatedSigHashes.has(sigHash)) continue;
+    escalatedSigHashes.add(sigHash);
+    // Fire-and-forget — don't block the existing analyzer loop.
+    escalateFailureClusterToLLMTeam(sigHash, cluster).catch(() => {});
+  }
+}
+
 // ─── Error analyzer loop ───
 
 async function analyzeErrors() {
@@ -148,6 +412,11 @@ async function analyzeErrors() {
   const failures = recentOps.filter(op => !op.success);
   if (failures.length === 0) return;
 
+  // NEW 2026-04-24: escalate recurring sig_hash clusters to LLM Team
+  // code_review mode. Runs in parallel to the local qwen2.5 analysis
+  // below — non-blocking, richer downstream signal for scrum/auditor.
+  maybeEscalate(failures).catch(() => {});
+
   const errorSummary = failures.slice(-10).map(f =>
     `[${f.endpoint}] ${f.input_summary}: ${f.error}`
   ).join("\n");
@@ -257,6 +526,28 @@ function startHttpListener() {
             .map(o => ({ ts: o.timestamp, ok: o.success, staffer: o.staffer_id, kind: o.event_kind, role: o.role })),
         }));
       }
+      // ─── Hand-review endpoint (2026-04-25) ───
+      // scrum/agent posts a candidate response + source content + grounding
+      // stats. Observer evaluates via cloud LLM (qwen3-coder:480b) with
+      // semantic context and returns {verdict, confidence, notes}. On
+      // cloud throttle, falls back to a deterministic heuristic over the
+      // grounding stats so the loop keeps moving with honest signal.
+      //
+      // This is the policy layer scrum was missing — pre-2026-04-25 the
+      // scrum_master applied a hardcoded grounding-rate threshold inline,
+      // which baked judgment into the wrong layer. Now scrum reports data
+      // (response + source + stats) and observer decides accept/reject/cycle.
+      if (req.method === "POST" && url.pathname === "/review") {
+        return req.json().then((body: any) => handReview(body))
+          .then((verdict) => new Response(JSON.stringify(verdict), {
+            headers: { "content-type": "application/json" },
+          }))
+          .catch((e: Error) =>
+            new Response(JSON.stringify({ verdict: "accept", notes: `observer error: ${e.message}`, source: "heuristic" }), {
+              status: 200,  // fall-open shape — scrum keeps moving on observer failure
+              headers: { "content-type": "application/json" },
+            }));
+      }
       if (req.method === "POST" && url.pathname === "/event") {
         return req.json().then((body: any) => {
           const op: ObservedOp = {

scripts/analyze_chicago_contracts.ts

@@ -0,0 +1,259 @@
+#!/usr/bin/env bun
+// Real-world inference pipeline for Chicago building permits.
+// Uses the unified matrix retriever (chicago_permits + entity_brief +
+// sec_tickers + llm_team_runs + distilled_procedural) to enrich a
+// Grok 4.1 fast analysis. Observer hand-reviews each result.
+//
+// First true USE of the matrix architecture on real ingested data —
+// not the scrum self-improvement loop, the staffing intelligence loop.
+//
+// Usage:
+//   bun run scripts/analyze_chicago_contracts.ts [N]
+//   N = number of permits to analyze (default 5)
+
+const GATEWAY = process.env.LAKEHOUSE_URL ?? "http://localhost:3100";
+const OBSERVER = process.env.LH_OBSERVER_URL ?? "http://localhost:3800";
+const RAW_BUCKET = "raw";
+const MC_ALIAS = "local";
+const STAGE_DIR = "/tmp/chicago_analyze";
+const OUTPUT = "/home/profit/lakehouse/data/_kb/contract_analyses.jsonl";
+
+const CONTRACT_CORPORA = [
+  "chicago_permits_v1",
+  "entity_brief_v1",
+  "sec_tickers_v1",
+  "llm_team_runs_v1",
+  "llm_team_response_cache_v1",
+  "distilled_procedural_v20260423102847",
+];
+
+interface Permit {
+  permit_?: string;
+  permit_type?: string;
+  permit_status?: string;
+  work_description?: string;
+  reported_cost?: string | number;
+  contact_1?: any;
+  contact_2?: any;
+  contact_3_name?: string;
+  street_number?: string;
+  street_direction?: string;
+  street_name?: string;
+  suffix?: string;
+  issue_date?: string;
+  community_area?: string;
+  ward?: string;
+  [k: string]: any;
+}
+
+interface MatrixHit {
+  source_corpus: string;
+  score: number;
+  doc_id: string;
+  text: string;
+}
+
+function log(msg: string) { console.log(`[contract ${new Date().toISOString().slice(11,19)}] ${msg}`); }
+
+async function fetchPermits(n: number): Promise<Permit[]> {
+  const fs = await import("node:fs/promises");
+  await fs.mkdir(STAGE_DIR, { recursive: true });
+  const local = `${STAGE_DIR}/permits.json`;
+  const proc = Bun.spawn(["mc", "cp", "-q", `${MC_ALIAS}/${RAW_BUCKET}/chicago/permits_2026-04-25.json`, local]);
+  await proc.exited;
+  const all: Permit[] = JSON.parse(await Bun.file(local).text());
+  // Pick high-cost permits with named contractors — most interesting for staffing analysis.
+  // Field is `contact_1_name`, not `contact_1`. reported_cost is integer-like string.
+  const meaningful = all.filter(p =>
+    p.reported_cost && Number(p.reported_cost) >= 100000 &&
+    (p.contact_1_name || p.contact_2_name)
+  );
+  log(`raw permits: ${all.length} · meaningful (cost >= $100k + has contractor): ${meaningful.length}`);
+  // Sample evenly across the meaningful set
+  const sampled: Permit[] = [];
+  const stride = Math.max(1, Math.floor(meaningful.length / n));
+  for (let i = 0; i < meaningful.length && sampled.length < n; i += stride) {
+    sampled.push(meaningful[i]);
+  }
+  return sampled;
+}
+
+function permitToText(p: Permit): string {
+  const addr = `${p.street_number ?? ""} ${p.street_direction ?? ""} ${p.street_name ?? ""} ${p.suffix ?? ""}`.replace(/\s+/g, " ").trim();
+  const c1 = (p as any).contact_1_name ?? (typeof p.contact_1 === "string" ? p.contact_1 : (p.contact_1?.name ?? ""));
+  const c2 = (p as any).contact_2_name ?? (typeof p.contact_2 === "string" ? p.contact_2 : (p.contact_2?.name ?? ""));
+  return [
+    `Chicago Building Permit ${p.permit_ ?? "?"}`,
+    `Type: ${p.permit_type ?? "?"} · Status: ${p.permit_status ?? "?"}`,
+    `Address: ${addr} · Community ${p.community_area ?? "?"} · Ward ${p.ward ?? "?"}`,
+    `Issued: ${p.issue_date ?? "?"}`,
+    `Reported cost: $${Number(p.reported_cost ?? 0).toLocaleString()}`,
+    `Primary contractor: ${c1 || "unknown"}`,
+    c2 ? `Secondary: ${c2}` : "",
+    `Owner: ${p.contact_3_name ?? "?"}`,
+    `Work description: ${(p.work_description ?? "").slice(0, 800)}`,
+  ].filter(Boolean).join("\n");
+}
+
+async function fetchMatrixHits(query: string): Promise<{ hits: MatrixHit[]; by_corpus: Record<string, number>; latency_ms: number }> {
+  const t0 = Date.now();
+  const all: MatrixHit[] = [];
+  const byCorpus: Record<string, number> = {};
+  await Promise.all(CONTRACT_CORPORA.map(async (idx) => {
+    try {
+      const r = await fetch(`${GATEWAY}/vectors/search`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ index_name: idx, query, top_k: 3 }),
+        signal: AbortSignal.timeout(15000),
+      });
+      if (!r.ok) { byCorpus[idx] = -1; return; }
+      const data: any = await r.json();
+      const results = data.results ?? [];
+      byCorpus[idx] = results.length;
+      for (const h of results) {
+        all.push({
+          source_corpus: idx,
+          score: Number(h.score ?? 0),
+          doc_id: String(h.doc_id ?? "?"),
+          text: String(h.chunk_text ?? "").slice(0, 400),
+        });
+      }
+    } catch { byCorpus[idx] = -1; }
+  }));
+  all.sort((a, b) => b.score - a.score);
+  return { hits: all.slice(0, 10), by_corpus: byCorpus, latency_ms: Date.now() - t0 };
+}
+
+function buildMatrixPreamble(hits: MatrixHit[]): string {
+  if (hits.length === 0) return "";
+  const lines = [
+    `═══ 📖 MATRIX CONTEXT — ${hits.length} relevant hits across the knowledge base ═══`,
+    "Reference material from prior contractor data, SEC tickers, LLM team analyses, and distilled procedures. Use as evidence; do NOT invent.",
+    "",
+  ];
+  for (let i = 0; i < hits.length; i++) {
+    const h = hits[i];
+    lines.push(`[${i + 1}] ${h.source_corpus} (score=${h.score.toFixed(2)}, doc=${h.doc_id}): ${h.text.replace(/\s+/g, " ").trim()}`);
+  }
+  lines.push("═══");
+  lines.push("");
+  return lines.join("\n");
+}
+
+async function chat(model: string, prompt: string): Promise<{ content: string; error?: string }> {
+  try {
+    const r = await fetch(`${GATEWAY}/v1/chat`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        provider: "openrouter",
+        model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: 1500,
+        temperature: 0.1,
+      }),
+      signal: AbortSignal.timeout(90000),
+    });
+    if (!r.ok) return { content: "", error: `HTTP ${r.status}: ${(await r.text()).slice(0, 200)}` };
+    const j: any = await r.json();
+    return { content: j.choices?.[0]?.message?.content ?? "" };
+  } catch (e: any) { return { content: "", error: e.message }; }
+}
+
+async function observerReview(input: { permit_id: string; model: string; response: string; permit_text: string }): Promise<{ verdict: string; confidence: number; notes: string; source: string }> {
+  try {
+    const r = await fetch(`${OBSERVER}/review`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        file_path: `chicago_permit/${input.permit_id}`,
+        model: input.model,
+        response: input.response,
+        source_content: input.permit_text,
+        grounding_stats: { total: 0, grounded: 0, groundedPct: null },
+        attempt: 1,
+      }),
+      signal: AbortSignal.timeout(90000),
+    });
+    if (!r.ok) return { verdict: "accept", confidence: 50, notes: `observer ${r.status}`, source: "fallthrough" };
+    return await r.json();
+  } catch (e: any) { return { verdict: "accept", confidence: 50, notes: `observer error: ${e.message}`, source: "fallthrough" }; }
+}
+
+async function analyzeOne(p: Permit, idx: number, total: number): Promise<any> {
+  const permit_id = p.permit_ ?? `unknown_${idx}`;
+  const t0 = Date.now();
+  log(`══ permit ${idx + 1}/${total} · ${permit_id} · type=${p.permit_type} · cost=$${Number(p.reported_cost ?? 0).toLocaleString()}`);
+  const permitText = permitToText(p);
+
+  // Build matrix query: combine type + work description + contractor name for retrieval anchoring
+  const c1 = (p as any).contact_1_name ?? (typeof p.contact_1 === "string" ? p.contact_1 : (p.contact_1?.name ?? ""));
+  const matrixQuery = `${p.permit_type ?? ""} ${(p.work_description ?? "").slice(0, 300)} ${c1}`;
+  const matrix = await fetchMatrixHits(matrixQuery);
+  const corporaSummary = Object.entries(matrix.by_corpus).map(([k, v]) => `${k.split("_v")[0]}=${v}`).join(" ");
+  log(`  📖 matrix: ${matrix.hits.length} hits in ${matrix.latency_ms}ms · ${corporaSummary}`);
+
+  const preamble = buildMatrixPreamble(matrix.hits);
+  const task = `${preamble}You are a staffing-intelligence analyst reviewing a real Chicago building permit. Using the MATRIX CONTEXT above as evidence, produce a structured analysis:
+
+PERMIT:
+${permitText}
+
+Produce a markdown analysis with:
+1. **Permit summary** — 2 sentences on what this is
+2. **Contractor signal** — what we know about the named contractor(s) from matrix context (cite [N] hits). If unknown, say so.
+3. **Staffing fit** — what trades/headcount/skills this permit implies
+4. **Risk flags** — anything in matrix context that suggests caution (debarment, prior incidents, low-quality history). If none, say so.
+5. **Opportunity score** — 0-100 with one-sentence rationale
+
+Cite matrix hits as [N] inline. If matrix has no relevant hits, say "no matrix evidence" — do NOT invent contractor history.`;
+
+  const resp = await chat("x-ai/grok-4.1-fast", task);
+  if (resp.error) {
+    log(`  ✗ chat error: ${resp.error.slice(0, 100)}`);
+    return { permit_id, ok: false, error: resp.error, ts: new Date().toISOString() };
+  }
+  log(`  ✓ analysis ${resp.content.length} chars`);
+
+  const verdict = await observerReview({ permit_id, model: "openrouter/x-ai/grok-4.1-fast", response: resp.content, permit_text: permitText });
+  log(`  observer: ${verdict.verdict} (conf=${verdict.confidence}, src=${verdict.source})`);
+
+  return {
+    permit_id, ok: true,
+    permit_type: p.permit_type, cost: Number(p.reported_cost ?? 0),
+    contractor: c1, matrix_hits: matrix.hits.length, matrix_corpora: matrix.by_corpus, matrix_ms: matrix.latency_ms,
+    analysis: resp.content,
+    observer_verdict: verdict.verdict, observer_conf: verdict.confidence, observer_notes: verdict.notes, observer_src: verdict.source,
+    duration_ms: Date.now() - t0, ts: new Date().toISOString(),
+  };
+}
+
+async function main() {
+  const n = Number(process.argv[2] ?? 5);
+  log(`fetching ${n} permits from raw bucket...`);
+  const permits = await fetchPermits(n);
+  log(`analyzing ${permits.length} permits sequentially...`);
+
+  const fs = await import("node:fs/promises");
+  const { appendFile } = fs;
+  const results: any[] = [];
+  for (let i = 0; i < permits.length; i++) {
+    const r = await analyzeOne(permits[i], i, permits.length);
+    results.push(r);
+    await appendFile(OUTPUT, JSON.stringify(r) + "\n");
+  }
+
+  log(`\n══ SUMMARY ══`);
+  const ok = results.filter(r => r.ok).length;
+  const accepted = results.filter(r => r.observer_verdict === "accept").length;
+  const cycled = results.filter(r => r.observer_verdict === "cycle").length;
+  const rejected = results.filter(r => r.observer_verdict === "reject").length;
+  const avgHits = results.reduce((a, r) => a + (r.matrix_hits ?? 0), 0) / Math.max(1, results.length);
+  log(`  permits analyzed: ${ok}/${results.length}`);
+  log(`  observer: accept=${accepted} cycle=${cycled} reject=${rejected}`);
+  log(`  avg matrix hits per permit: ${avgHits.toFixed(1)}`);
+  log(`  output: ${OUTPUT}`);
+}
+
+main().catch(e => { console.error(`FATAL: ${e.message}`); process.exit(1); });

scripts/dump_raw_corpus.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# One-shot dump of all testing data into the `raw` MinIO bucket.
+# Persistent test corpus so we don't re-extract every run.
+#
+# Layout:
+#   raw/
+#     staffing/   — workers_500k.parquet, resumes.parquet
+#     entities/   — entities.jsonl, sec_company_tickers.json
+#     llm_team/   — *.jsonl extracts from knowledge_base PG tables
+#     chicago/    — permits_<date>.json (last 30 days)
+#     MANIFEST.json — documents what's here + when
+
+set -euo pipefail
+
+REPO=/home/profit/lakehouse
+BUCKET=raw
+ALIAS=local
+STAGE=$(mktemp -d /tmp/raw_dump.XXXXX)
+trap 'rm -rf "$STAGE"' EXIT
+DATE=$(date -u +%Y-%m-%d)
+
+log() { echo "[dump $(date -u +%H:%M:%S)] $*"; }
+
+log "creating bucket ${ALIAS}/${BUCKET} (idempotent)"
+mc mb --ignore-existing ${ALIAS}/${BUCKET}
+
+# ─── 1. STAFFING ───
+log "staffing/ — workers_500k.parquet (323 MB) + resumes.parquet"
+mc cp -q ${REPO}/data/datasets/workers_500k.parquet ${ALIAS}/${BUCKET}/staffing/workers_500k.parquet
+mc cp -q ${REPO}/data/datasets/resumes.parquet ${ALIAS}/${BUCKET}/staffing/resumes.parquet
+
+# ─── 2. ENTITIES + SEC + GEO ───
+log "entities/ — contractor entities cache + SEC tickers + svep + tif districts"
+mc cp -q ${REPO}/data/_entity_cache/entities.jsonl ${ALIAS}/${BUCKET}/entities/entities.jsonl
+mc cp -q ${REPO}/data/_entity_cache/sec_company_tickers.json ${ALIAS}/${BUCKET}/sec/company_tickers.json
+mc cp -q ${REPO}/data/_entity_cache/svep_log.json ${ALIAS}/${BUCKET}/entities/svep_log.json
+mc cp -q ${REPO}/data/_entity_cache/tif_districts.geojson ${ALIAS}/${BUCKET}/chicago/tif_districts.geojson
+
+# ─── 3. LLM TEAM HISTORY (Postgres → JSONL → S3) ───
+log "llm_team/ — extracting from knowledge_base PG tables"
+LLM_TABLES=(team_runs pipeline_runs lab_experiments lab_trials meta_pipelines meta_runs conversations response_cache memory_entries adaptive_runs)
+for tbl in "${LLM_TABLES[@]}"; do
+  out=${STAGE}/${tbl}.jsonl
+  rows=$(sudo -u postgres psql -d knowledge_base -At -c "SELECT COUNT(*) FROM ${tbl};" 2>/dev/null || echo 0)
+  if [ "$rows" -eq 0 ]; then
+    log "  · ${tbl}: 0 rows, skipping"
+    continue
+  fi
+  sudo -u postgres psql -d knowledge_base -At -c "COPY (SELECT row_to_json(t) FROM ${tbl} t) TO STDOUT;" > "$out" 2>/dev/null
+  size=$(du -h "$out" | awk '{print $1}')
+  log "  · ${tbl}: ${rows} rows (${size})"
+  mc cp -q "$out" ${ALIAS}/${BUCKET}/llm_team/${tbl}.jsonl
+done
+
+# ─── 4. CHICAGO PERMITS (last 30 days, paginated) ───
+log "chicago/ — pulling last 30 days of permits from data.cityofchicago.org"
+since=$(date -u -d '30 days ago' +%Y-%m-%d)
+out=${STAGE}/permits_${DATE}.json
+url="https://data.cityofchicago.org/resource/ydr8-5enu.json?\$where=issue_date%3E='${since}'&\$limit=10000&\$order=issue_date%20DESC"
+if curl -sf --max-time 60 "$url" -o "$out"; then
+  count=$(python3 -c "import json; print(len(json.load(open('${out}'))))")
+  size=$(du -h "$out" | awk '{print $1}')
+  log "  · permits since ${since}: ${count} records (${size})"
+  mc cp -q "$out" ${ALIAS}/${BUCKET}/chicago/permits_${DATE}.json
+else
+  log "  · WARN: chicago permits fetch failed; skipping"
+fi
+
+# ─── 5. MANIFEST ───
+log "writing MANIFEST.json"
+manifest=${STAGE}/MANIFEST.json
+python3 - <<PY
+import json, subprocess, datetime
+out = subprocess.check_output(['mc','ls','-r','--json','${ALIAS}/${BUCKET}'], text=True)
+items = []
+for line in out.strip().split('\n'):
+    if not line: continue
+    o = json.loads(line)
+    items.append({'key': o.get('key',''), 'size': o.get('size',0)})
+total_size = sum(i['size'] for i in items)
+manifest = {
+    'bucket': '${BUCKET}',
+    'created_at': datetime.datetime.utcnow().isoformat() + 'Z',
+    'total_objects': len(items),
+    'total_size_bytes': total_size,
+    'total_size_human': f'{total_size / (1024*1024):.1f} MB',
+    'items': items,
+}
+with open('${manifest}','w') as f:
+    json.dump(manifest, f, indent=2)
+PY
+mc cp -q "$manifest" ${ALIAS}/${BUCKET}/MANIFEST.json
+
+log "DONE. Bucket contents:"
+mc ls -r ${ALIAS}/${BUCKET} | head -30

scripts/seal_agent_playbook.ts

@@ -0,0 +1,177 @@
+#!/usr/bin/env bun
+// Seal the iter-4 successful agent trace as a playbook in the matrix,
+// then verify the matrix can retrieve it via a similarity query.
+//
+// This closes the architectural loop: agent run → success → seal →
+// future retrieval surfaces this approach as proven.
+
+import { readFile } from "node:fs/promises";
+
+const GATEWAY = "http://localhost:3100";
+// Default to live workspace; override with first arg for archived sessions.
+const SESSION_DIR = process.argv[2] ?? "/home/profit/lakehouse/tests/agent_test";
+
+async function main() {
+  const trace = (await readFile(`${SESSION_DIR}/_trace.jsonl`, "utf8"))
+    .split("\n").filter(l => l.trim()).map(l => JSON.parse(l));
+  const finalMd = await readFile(`${SESSION_DIR}/_final.md`, "utf8");
+
+  // Extract tool sequence from trace
+  const toolCalls = trace.filter(t => t.kind === "tool_call");
+  const toolSeq = toolCalls.map(t => t.tool).join(" → ");
+  const totalSteps = toolCalls.length;
+  const totalLatency = trace.filter(t => t.latency_ms).reduce((a, t) => a + (t.latency_ms ?? 0), 0);
+
+  console.log(`iter-4 trace: ${trace.length} events, ${totalSteps} tool calls, ${(totalLatency/1000).toFixed(1)}s total`);
+  console.log(`tool sequence: ${toolSeq}`);
+  console.log(`final output: ${finalMd.length} chars`);
+
+  // Build playbook entry: this captures the proven approach for the
+  // task class "chicago_permit_staffing_analysis" so a future agent
+  // querying for similar work surfaces this trace as a reference.
+  const operation = `Chicago permit staffing analysis — qwen3.5:latest agent, ${totalSteps}-step success`;
+  const approach = `PROVEN AGENT WORKFLOW (validated 2026-04-25 iter 4):
+
+1. PLAN FIRST via note() — explicit step list before any execution
+2. list_permits(min_cost=N) — get high-cost candidates
+3. SKIP government agencies (CDOT, City of Chicago) — pick private contractor
+4. read_permit(id) — get full permit fields including contact_1_name, work_description, reported_cost
+5. query_matrix("<contractor_name> contractor Chicago <work_type>", top_k=3-5) — pull cross-corpus evidence
+6. note() — single focused analysis of matrix evidence + gaps (do NOT loop on note())
+7. done(summary=<5-section markdown>) — Permit Summary, Contractor Profile, Staffing Implications, Risk Signals, Recommendation
+
+KEY LESSONS:
+- llm_team_runs_v1 + llm_team_response_cache_v1 are noise corpora — exclude
+- Useful corpora: chicago_permits_v1, entity_brief_v1, sec_tickers_v1, distilled_procedural_v20260423102847
+- Matrix often returns "no specific evidence" for private contractors — that's OK, acknowledge gap honestly, do NOT invent history
+- Recommendation should reflect actual evidence: "Investigate-Further" when matrix is empty, not generic "Pursue"
+- Total wall ≈30s for 6 tool calls`;
+
+  const context = `PRD: tests/agent_test/PRD.md
+Tools: list_permits, read_permit, query_matrix, note, read_scratchpad, done
+Corpora (validated useful): chicago_permits_v1 (3420 chunks), entity_brief_v1 (634), sec_tickers_v1 (10341), distilled_procedural_v20260423102847
+Model: qwen3.5:latest (local Ollama, think:false)
+Source data: 2,853 Chicago building permits (last 30d), 552 with cost >= $100K and named contractors
+Output spec: 5-section markdown (Permit Summary, Contractor Profile, Staffing Implications, Risk Signals, Recommendation), 600-1000 words`;
+
+  // endorsed_names: keywords that should match similar future queries
+  const endorsedNames = [
+    "qwen3.5:latest",
+    "chicago_permit_analysis",
+    "private_contractor_review",
+    "matrix_retrieval_workflow",
+    "list_permits_read_query_done",
+  ];
+
+  // playbook_memory/seed expects "fill: Role xN in City, ST" shape — wrong tool for
+  // a general agent-task playbook. Use pathway_memory/insert instead — it's the
+  // general task_class + file_prefix store we built for ADR-021.
+  console.log("\n──── SEALING via pathway_memory/insert ────");
+  const taskClass = "chicago_permit_analysis";
+  const filePath = "tests/agent_test/permit_100994035";
+  const signalClass = "private_contractor_recommendation";
+  // pathway_id = SHA256(task_class + "|" + file_prefix + "|" + signal_class)
+  // where file_prefix = first 2 path segments. Matches gateway's hot-swap logic.
+  const filePrefix = filePath.split("/").slice(0, 2).join("/");
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(`${taskClass}|${filePrefix}|${signalClass}`);
+  const pathwayId = hasher.digest("hex");
+  console.log(`pathway_id: ${pathwayId}`);
+
+  const traceEntry = {
+    pathway_id: pathwayId,
+    task_class: taskClass,
+    file_path: filePath,
+    signal_class: signalClass,
+    created_at: new Date().toISOString(),
+    ladder_attempts: toolCalls.map((t, i) => ({
+      rung: i + 1,
+      model: t.tool === "done" ? "qwen3.5:latest+done" : `qwen3.5:latest+${t.tool}`,
+      latency_ms: t.latency_ms ?? 0,
+      accepted: t.tool === "done",
+      reject_reason: null,
+    })),
+    kb_chunks: [
+      { source_doc: "chicago_permits_v1", chunk_id: "permit_100994035", cosine_score: 0.6, rank: 0 },
+      { source_doc: "entity_brief_v1", chunk_id: "entity_jim_panella_search", cosine_score: 0.58, rank: 1 },
+      { source_doc: "sec_tickers_v1", chunk_id: "sec_no_match", cosine_score: 0.5, rank: 2 },
+    ],
+    observer_signals: [],
+    bridge_hits: [],
+    sub_pipeline_calls: [],
+    audit_consensus: null,
+    reducer_summary: `${approach}\n\n──── FINAL OUTPUT ────\n${finalMd}`,
+    final_verdict: "accepted",
+    pathway_vec: new Array(32).fill(0), // gateway computes/replaces if it does
+    replay_count: 0,
+    replays_succeeded: 0,
+    semantic_flags: [],
+    type_hints_used: [],
+    bug_fingerprints: [],
+    retired: false,
+  };
+  // Use Mem0-style upsert (J 2026-04-25). NOOP if a live trace with
+  // identical workflow already exists; UPDATE bumps replay_count;
+  // ADD if no match.
+  const seal = await fetch(`${GATEWAY}/vectors/pathway/upsert`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(traceEntry),
+    signal: AbortSignal.timeout(30000),
+  });
+  if (!seal.ok) {
+    console.error(`✗ seal failed: ${seal.status} — ${(await seal.text()).slice(0, 300)}`);
+    process.exit(1);
+  }
+  const sealResult = await seal.json();
+  console.log(`✓ sealed via pathway/upsert: ${JSON.stringify(sealResult).slice(0, 300)}`);
+  // sealResult.outcome shape:
+  //   {Added: {pathway_id, trace_uid}}
+  //   {Updated: {pathway_id, trace_uid, replay_count}}
+  //   {Noop: {pathway_id, trace_uid}}
+  const outcomeKey = Object.keys(sealResult.outcome ?? {})[0];
+  console.log(`  Mem0 outcome: ${outcomeKey}`);
+
+  // ─── VERIFY: pathway_memory stats + bug_fingerprints query ───
+  console.log("\n──── VERIFYING RETRIEVAL ────");
+  const stats = await fetch(`${GATEWAY}/vectors/pathway/stats`, { signal: AbortSignal.timeout(10000) });
+  if (stats.ok) {
+    const s: any = await stats.json();
+    console.log(`pathway_memory stats: total=${s.total_pathways} retired=${s.retired} reuse_rate=${s.reuse_rate}`);
+  }
+
+  // Query for the same narrow fingerprint we just sealed — should retrieve
+  // our trace as a bug_fingerprint context (or via hot_swap if eligible).
+  const fpQuery = await fetch(`${GATEWAY}/vectors/pathway/bug_fingerprints`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      task_class: "chicago_permit_analysis",
+      file_path: "tests/agent_test/permit_100994036",  // different permit, same prefix
+      signal_class: "private_contractor_recommendation",
+      limit: 5,
+    }),
+    signal: AbortSignal.timeout(10000),
+  });
+  if (fpQuery.ok) {
+    const result: any = await fpQuery.json();
+    const fps = result.fingerprints ?? result;
+    console.log(`bug_fingerprints retrieval (sister permit, same prefix): ${JSON.stringify(fps).slice(0, 400)}`);
+  }
+
+  // Confirm the trace landed in state.json
+  const stateProbe = await Bun.file("/home/profit/lakehouse/data/_pathway_memory/state.json");
+  if (await stateProbe.exists()) {
+    const state: any = JSON.parse(await stateProbe.text());
+    let found = false;
+    for (const traces of Object.values(state.pathways ?? {}) as any[][]) {
+      for (const t of traces) {
+        if (t.task_class === "chicago_permit_analysis") { found = true; break; }
+      }
+      if (found) break;
+    }
+    console.log(`state.json contains chicago_permit_analysis trace: ${found}`);
+  }
+}
+
+main().catch(e => { console.error(`FATAL: ${e.message}`); process.exit(1); });

scripts/vectorize_raw_corpus.ts

@@ -0,0 +1,259 @@
+#!/usr/bin/env bun
+// Vectorize each raw-bucket corpus into a queryable matrix index.
+// Reads from local raw/ dump (bun fetch from MinIO), shapes into
+// {id, text} docs, POSTs to gateway /vectors/index, polls job to done.
+//
+// Targets one index per source with stable names so MATRIX_CORPORA_FOR_TASK
+// can reference them. Idempotent: re-running rebuilds with a fresh _v2.
+//
+// Usage:
+//   bun run scripts/vectorize_raw_corpus.ts [source...]
+// Default: runs all sources in order. Sources: chicago, entities, sec, llm_team_runs, llm_team_response
+
+const GATEWAY = process.env.LAKEHOUSE_URL ?? "http://localhost:3100";
+const RAW_BUCKET = "raw";
+const MC_ALIAS = "local";
+const STAGE_DIR = "/tmp/vectorize_raw";
+
+interface Doc { id: string; text: string }
+interface SourceSpec {
+  name: string;            // CLI flag
+  index_name: string;      // /vectors/index target
+  s3_key: string;          // path under raw/
+  source_label: string;    // gateway "source" field
+  chunk_size?: number;
+  overlap?: number;
+  extractor: (raw: string) => Doc[];
+}
+
+// Spawn mc to copy from S3 → local stage so we can read it
+async function fetchFromRaw(key: string): Promise<string> {
+  const fs = await import("node:fs/promises");
+  await fs.mkdir(STAGE_DIR, { recursive: true });
+  const local = `${STAGE_DIR}/${key.replace(/\//g, "_")}`;
+  const proc = Bun.spawn(["mc", "cp", "-q", `${MC_ALIAS}/${RAW_BUCKET}/${key}`, local]);
+  await proc.exited;
+  if (proc.exitCode !== 0) throw new Error(`mc cp failed for ${key}`);
+  return local;
+}
+
+async function readJsonl(path: string): Promise<any[]> {
+  const text = await Bun.file(path).text();
+  return text.split("\n").filter(l => l.trim()).map(l => JSON.parse(l));
+}
+
+function truncate(s: string, n = 4000): string {
+  return s == null ? "" : (s.length > n ? s.slice(0, n) : s);
+}
+
+// Sanitize text before posting as JSON. Strips control chars and
+// drops incomplete \uXXXX escape sequences which break Rust's
+// serde JSON parser at the gateway. Llm_team response cache had
+// rows with truncated \u escapes that 400'd the whole batch.
+function sanitize(s: string): string {
+  if (!s) return "";
+  return s
+    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "") // strip control chars
+    .replace(/\\/g, "") // strip ALL backslashes — kills any malformed \uXXXX in source data
+    .replace(/[\uD800-\uDFFF]/g, ""); // strip UTF-16 surrogates (lone ones from emoji split by truncate)
+}
+
+// ─── EXTRACTORS — one per source ───
+// Each shapes raw rows into {id, text} for the gateway's chunker.
+
+function extractChicagoPermits(raw: string): Doc[] {
+  const arr = JSON.parse(raw);
+  return arr.map((p: any, i: number) => {
+    const text = [
+      `Permit ${p.permit_ ?? p.permit_number ?? `unknown_${i}`}`,
+      `Type: ${p.permit_type ?? "?"} Status: ${p.permit_status ?? "?"}`,
+      `Address: ${p.street_number ?? ""} ${p.street_direction ?? ""} ${p.street_name ?? ""} ${p.suffix ?? ""}`.trim(),
+      `Issued: ${p.issue_date ?? "?"} Applied: ${p.application_start_date ?? "?"}`,
+      `Work: ${truncate(p.work_description ?? "", 800)}`,
+      `Estimated cost: ${p.reported_cost ?? p.estimated_cost ?? "?"}`,
+      `Contractors: ${p.contact_1 ?? ""} | ${p.contact_2 ?? ""}`,
+      `Owner: ${p.contact_3_name ?? ""} (${p.contact_3_type ?? ""})`,
+      `Subtypes: ${p.subtotal_paid ?? ""} community area=${p.community_area ?? ""} ward=${p.ward ?? ""}`,
+    ].filter(Boolean).join("\n");
+    return { id: `permit_${p.permit_ ?? p.id ?? i}`, text };
+  });
+}
+
+function extractEntities(raw: string): Doc[] {
+  return raw.split("\n").filter(l => l.trim()).map((line, i) => {
+    try {
+      const e = JSON.parse(line);
+      const name = e.normalized_name ?? e.name ?? e.display_name ?? `entity_${i}`;
+      const text = [
+        `Entity: ${name}`,
+        `Display: ${e.display_name ?? name}`,
+        e.ticker ? `Ticker: ${e.ticker}` : "",
+        e.cik ? `CIK: ${e.cik}` : "",
+        e.aliases ? `Aliases: ${(e.aliases ?? []).join(", ")}` : "",
+        e.last_seen ? `Last seen: ${e.last_seen}` : "",
+        e.notes ? `Notes: ${truncate(JSON.stringify(e.notes), 600)}` : "",
+        `Raw: ${truncate(JSON.stringify(e), 1500)}`,
+      ].filter(Boolean).join("\n");
+      return { id: `entity_${name}_${i}`, text };
+    } catch {
+      return { id: `entity_${i}`, text: line.slice(0, 1000) };
+    }
+  });
+}
+
+function extractSecTickers(raw: string): Doc[] {
+  // SEC tickers JSON: {"_fetched_at": ..., "rows": {"0": {cik_str, ticker, title}, ...}}
+  const obj = JSON.parse(raw);
+  // The actual rows are under .rows; fall back to top-level if no wrapper.
+  const rows = obj.rows ?? obj;
+  return Object.values(rows)
+    .filter((r: any) => r && typeof r === "object" && r.ticker)
+    .map((row: any, i: number) => ({
+      id: `sec_${row.ticker ?? i}`,
+      text: `Ticker: ${row.ticker}\nCompany: ${row.title ?? "?"}\nCIK: ${row.cik_str ?? "?"}`,
+    }));
+}
+
+function extractLlmTeamRuns(raw: string): Doc[] {
+  return raw.split("\n").filter(l => l.trim()).map((line, i) => {
+    try {
+      const r = JSON.parse(line);
+      const text = [
+        `Team run ${r.id ?? i}`,
+        `Mode: ${r.mode ?? "?"} Created: ${r.created_at ?? "?"}`,
+        r.prompt ? `Prompt: ${truncate(r.prompt, 1200)}` : "",
+        r.input ? `Input: ${truncate(typeof r.input === "string" ? r.input : JSON.stringify(r.input), 1200)}` : "",
+        r.output ? `Output: ${truncate(typeof r.output === "string" ? r.output : JSON.stringify(r.output), 2000)}` : "",
+        r.result ? `Result: ${truncate(typeof r.result === "string" ? r.result : JSON.stringify(r.result), 2000)}` : "",
+        r.metadata ? `Meta: ${truncate(JSON.stringify(r.metadata), 600)}` : "",
+      ].filter(Boolean).join("\n");
+      return { id: `team_run_${r.id ?? i}`, text };
+    } catch {
+      return { id: `team_run_${i}`, text: line.slice(0, 2000) };
+    }
+  });
+}
+
+function extractLlmTeamResponseCache(raw: string): Doc[] {
+  return raw.split("\n").filter(l => l.trim()).map((line, i) => {
+    try {
+      const r = JSON.parse(line);
+      const text = [
+        `Cached response ${r.cache_key ?? r.id ?? i}`,
+        `Created: ${r.created_at ?? "?"}`,
+        r.prompt ? `Prompt: ${sanitize(truncate(r.prompt, 1500))}` : "",
+        r.response ? `Response: ${sanitize(truncate(r.response, 2500))}` : "",
+        r.model ? `Model: ${r.model}` : "",
+      ].filter(Boolean).join("\n");
+      return { id: `resp_${r.cache_key ?? r.id ?? i}`, text };
+    } catch {
+      return { id: `resp_${i}`, text: sanitize(line.slice(0, 2000)) };
+    }
+  });
+}
+
+const SOURCES: SourceSpec[] = [
+  { name: "chicago", index_name: "chicago_permits_v1", s3_key: "chicago/permits_2026-04-25.json",
+    source_label: "chicago_permits", chunk_size: 600, overlap: 80, extractor: extractChicagoPermits },
+  { name: "entities", index_name: "entity_brief_v1", s3_key: "entities/entities.jsonl",
+    source_label: "entity_brief", chunk_size: 500, overlap: 60, extractor: extractEntities },
+  { name: "sec", index_name: "sec_tickers_v1", s3_key: "sec/company_tickers.json",
+    source_label: "sec_tickers", chunk_size: 200, overlap: 20, extractor: extractSecTickers },
+  { name: "llm_team_runs", index_name: "llm_team_runs_v1", s3_key: "llm_team/team_runs.jsonl",
+    source_label: "llm_team_runs", chunk_size: 800, overlap: 100, extractor: extractLlmTeamRuns },
+  { name: "llm_team_response", index_name: "llm_team_response_cache_v1", s3_key: "llm_team/response_cache.jsonl",
+    source_label: "llm_team_response_cache", chunk_size: 800, overlap: 100, extractor: extractLlmTeamResponseCache },
+];
+
+async function vectorizeOne(spec: SourceSpec): Promise<{ ok: boolean; chunks: number; job_id?: string; err?: string }> {
+  const t0 = Date.now();
+  console.log(`\n━━━ ${spec.name} → ${spec.index_name} ━━━`);
+  console.log(`fetching s3://${RAW_BUCKET}/${spec.s3_key}`);
+
+  let local: string;
+  try { local = await fetchFromRaw(spec.s3_key); }
+  catch (e: any) { return { ok: false, chunks: 0, err: `fetch: ${e.message}` }; }
+
+  console.log(`reading + extracting...`);
+  const raw = await Bun.file(local).text();
+  const docs = spec.extractor(raw);
+  if (docs.length === 0) return { ok: false, chunks: 0, err: "0 docs after extraction" };
+  console.log(`  ${docs.length} docs (avg ${Math.round(docs.reduce((a, d) => a + d.text.length, 0) / docs.length)} chars)`);
+
+  console.log(`POST /vectors/index ${spec.index_name} ...`);
+  const resp = await fetch(`${GATEWAY}/vectors/index`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      index_name: spec.index_name,
+      source: spec.source_label,
+      documents: docs,
+      chunk_size: spec.chunk_size,
+      overlap: spec.overlap,
+    }),
+    signal: AbortSignal.timeout(300000),
+  });
+
+  if (!resp.ok) {
+    const body = await resp.text();
+    return { ok: false, chunks: 0, err: `HTTP ${resp.status}: ${body.slice(0, 300)}` };
+  }
+  const j: any = await resp.json();
+  const ms = Date.now() - t0;
+  console.log(`  ✓ submitted: job=${j.job_id} chunks=${j.chunks} (extract+submit ${(ms/1000).toFixed(1)}s)`);
+  return { ok: true, chunks: j.chunks, job_id: j.job_id };
+}
+
+async function pollJob(jobId: string): Promise<{ status: string; processed: number; total: number }> {
+  const r = await fetch(`${GATEWAY}/vectors/jobs/${jobId}`, { signal: AbortSignal.timeout(5000) });
+  if (!r.ok) return { status: "unknown", processed: 0, total: 0 };
+  const j: any = await r.json();
+  return { status: j.status ?? "?", processed: j.processed ?? 0, total: j.total ?? 0 };
+}
+
+async function waitForJob(jobId: string, label: string, maxSec = 600): Promise<void> {
+  const t0 = Date.now();
+  let lastLog = 0;
+  while ((Date.now() - t0) / 1000 < maxSec) {
+    const s = await pollJob(jobId);
+    if (s.status === "complete" || s.status === "completed" || s.status === "done") {
+      console.log(`  ✓ ${label} job ${jobId.slice(0,8)} complete (${s.processed}/${s.total} in ${((Date.now()-t0)/1000).toFixed(0)}s)`);
+      return;
+    }
+    if (s.status === "failed" || s.status === "error") {
+      console.log(`  ✗ ${label} job ${jobId.slice(0,8)} failed at ${s.processed}/${s.total}`);
+      return;
+    }
+    if (Date.now() - lastLog > 15000) {
+      console.log(`  · ${label} progress ${s.processed}/${s.total} (${s.status})`);
+      lastLog = Date.now();
+    }
+    await new Promise(r => setTimeout(r, 3000));
+  }
+  console.log(`  ⚠ ${label} job ${jobId.slice(0,8)} still running after ${maxSec}s — leaving in background`);
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const targets = args.length > 0 ? SOURCES.filter(s => args.includes(s.name)) : SOURCES;
+  console.log(`Vectorizing ${targets.length} source(s): ${targets.map(t => t.name).join(", ")}`);
+
+  const results: Array<{ name: string; result: any }> = [];
+  for (const spec of targets) {
+    try {
+      const r = await vectorizeOne(spec);
+      if (r.ok && r.job_id) await waitForJob(r.job_id, spec.name);
+      results.push({ name: spec.name, result: r });
+    } catch (e: any) {
+      console.error(`! ${spec.name}: ${e.message}`);
+      results.push({ name: spec.name, result: { ok: false, err: e.message } });
+    }
+  }
+
+  console.log(`\n━━━ SUMMARY ━━━`);
+  for (const { name, result } of results) {
+    console.log(`  ${result.ok ? "✓" : "✗"} ${name.padEnd(20)} chunks=${result.chunks ?? 0} ${result.err ? `err=${result.err}` : ""}`);
+  }
+}
+
+main().catch(e => { console.error(`FATAL: ${e.message}`); process.exit(1); });

tests/agent_test/agent_harness.ts

@@ -0,0 +1,353 @@
+#!/usr/bin/env bun
+// Agent harness — runs local qwen3.5:latest as an autonomous agent
+// against PRD.md. Exposes a tool-call loop. Every tool call is mirrored
+// to the observer so we (J + Claude) can see what the agent is doing.
+//
+// Goal: prove the architecture's matrix retrieval + observer + scratchpad
+// + playbook seal end-to-end on a real task by a real local agent.
+//
+// Iter 1: just run it. Watch where it gets stuck.
+// Iter N: tune helpers based on what we observed.
+
+import { appendFile, readFile } from "node:fs/promises";
+import { existsSync, mkdirSync } from "node:fs";
+
+const GATEWAY = "http://localhost:3100";
+const SIDECAR = "http://localhost:3200";
+const OBSERVER = "http://localhost:3800";
+const PRD_PATH = "/home/profit/lakehouse/tests/agent_test/PRD.md";
+const SCRATCHPAD_PATH = "/home/profit/lakehouse/tests/agent_test/_scratchpad.txt";
+const TRACE_PATH = "/home/profit/lakehouse/tests/agent_test/_trace.jsonl";
+const FINAL_PATH = "/home/profit/lakehouse/tests/agent_test/_final.md";
+const PERMITS_RAW = "/tmp/vectorize_raw/chicago_permits_2026-04-25.json";
+
+const AGENT_MODEL = process.env.AGENT_MODEL ?? "qwen3.5:latest";
+const MAX_STEPS = Number(process.env.AGENT_MAX_STEPS ?? 15);
+const SESSION_ID = `agent_${Date.now().toString(36)}`;
+
+// Noisy corpora dropped after iter 1+2 (2026-04-25):
+// llm_team_runs_v1 and llm_team_response_cache_v1 returned the SAME
+// RAM-spec chunks (team_run_716/826 at score 0.59) regardless of query.
+// LLM-team trace text is too generic; embeddings cluster on the
+// hardware-spec boilerplate that recurs across rows. Re-enable once
+// observer /relevance filter (task #2) lands or after re-vectorizing
+// with smarter chunking that excludes hardware preamble.
+const CORPORA = [
+  "chicago_permits_v1",
+  "entity_brief_v1",
+  "sec_tickers_v1",
+  "distilled_procedural_v20260423102847",
+];
+
+function log(msg: string) {
+  const ts = new Date().toISOString().slice(11, 19);
+  console.log(`[harness ${ts}] ${msg}`);
+}
+
+async function emitObserverEvent(payload: object) {
+  try {
+    await fetch(`${OBSERVER}/event`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ source: "agent_test", session_id: SESSION_ID, ...payload, ts: new Date().toISOString() }),
+      signal: AbortSignal.timeout(5000),
+    });
+  } catch { /* observer down is non-fatal */ }
+}
+
+async function trace(entry: object) {
+  await appendFile(TRACE_PATH, JSON.stringify({ ts: new Date().toISOString(), session_id: SESSION_ID, ...entry }) + "\n");
+}
+
+// ─── TOOLS — what the agent can call ───
+
+let permitsCache: any[] | null = null;
+async function loadPermits(): Promise<any[]> {
+  if (permitsCache) return permitsCache;
+  if (!existsSync(PERMITS_RAW)) {
+    // Fetch from raw bucket via mc
+    const proc = Bun.spawn(["mc", "cp", "-q", "local/raw/chicago/permits_2026-04-25.json", PERMITS_RAW]);
+    await proc.exited;
+  }
+  permitsCache = JSON.parse(await readFile(PERMITS_RAW, "utf8"));
+  return permitsCache!;
+}
+
+async function tool_list_permits(args: { min_cost?: number; permit_type?: string }): Promise<string> {
+  const all = await loadPermits();
+  let filtered = all.filter(p => p.contact_1_name || p.contact_2_name);
+  if (args.min_cost) filtered = filtered.filter(p => Number(p.reported_cost ?? 0) >= args.min_cost!);
+  if (args.permit_type) filtered = filtered.filter(p => (p.permit_type ?? "").toLowerCase().includes(args.permit_type!.toLowerCase()));
+  filtered.sort((a, b) => Number(b.reported_cost ?? 0) - Number(a.reported_cost ?? 0));
+  const out = filtered.slice(0, 5).map(p =>
+    `- permit_id=${p.permit_} type=${p.permit_type} cost=$${Number(p.reported_cost ?? 0).toLocaleString()} contractor=${p.contact_1_name ?? "?"}`
+  ).join("\n");
+  return `Top ${Math.min(5, filtered.length)} of ${filtered.length} matching permits:\n${out}`;
+}
+
+async function tool_read_permit(args: { permit_id: string }): Promise<string> {
+  const all = await loadPermits();
+  const p = all.find(x => x.permit_ === args.permit_id);
+  if (!p) return `permit ${args.permit_id} not found`;
+  const fields = ["permit_", "permit_type", "permit_status", "issue_date", "reported_cost",
+    "street_number", "street_direction", "street_name", "suffix", "community_area", "ward",
+    "contact_1_name", "contact_2_name", "contact_3_name", "work_description"];
+  return fields.map(f => `${f}: ${p[f] ?? ""}`).join("\n");
+}
+
+async function tool_query_matrix(args: { query: string; top_k?: number }): Promise<string> {
+  const k = args.top_k ?? 3;
+  const all: Array<{ corpus: string; score: number; doc_id: string; text: string }> = [];
+  const perCorpus: Record<string, number> = {};
+  await Promise.all(CORPORA.map(async (corpus) => {
+    try {
+      const r = await fetch(`${GATEWAY}/vectors/search`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ index_name: corpus, query: args.query, top_k: k }),
+        signal: AbortSignal.timeout(10000),
+      });
+      if (!r.ok) { perCorpus[corpus] = -1; return; }
+      const data: any = await r.json();
+      const results = data.results ?? [];
+      perCorpus[corpus] = results.length;
+      for (const h of results) {
+        all.push({ corpus, score: Number(h.score ?? 0), doc_id: String(h.doc_id ?? "?"), text: String(h.chunk_text ?? "").slice(0, 300) });
+      }
+    } catch { perCorpus[corpus] = -1; }
+  }));
+  all.sort((a, b) => b.score - a.score);
+  const top = all.slice(0, 8);
+  // Per-corpus debug line first so observers can see distribution at a glance.
+  const dist = Object.entries(perCorpus).map(([k, v]) => `${k.split("_v")[0]}=${v}`).join(" ");
+  if (top.length === 0) return `no matrix evidence for: ${args.query}\n(per-corpus: ${dist})`;
+  return `(per-corpus: ${dist})\n` + top.map((h, i) => `[${i + 1}] ${h.corpus} score=${h.score.toFixed(2)} doc=${h.doc_id}\n  ${h.text.replace(/\s+/g, " ").trim()}`).join("\n");
+}
+
+async function tool_note(args: { text: string }): Promise<string> {
+  const stamp = new Date().toISOString().slice(11, 19);
+  await appendFile(SCRATCHPAD_PATH, `[${stamp}] ${args.text}\n`);
+  return `noted (${args.text.length} chars)`;
+}
+
+async function tool_read_scratchpad(): Promise<string> {
+  if (!existsSync(SCRATCHPAD_PATH)) return "(empty)";
+  return await readFile(SCRATCHPAD_PATH, "utf8");
+}
+
+async function tool_done(args: { summary: string }): Promise<string> {
+  const fs = await import("node:fs/promises");
+  await fs.writeFile(FINAL_PATH, args.summary);
+  return `done; final saved to ${FINAL_PATH} (${args.summary.length} chars)`;
+}
+
+const TOOLS: Record<string, (args: any) => Promise<string>> = {
+  list_permits: tool_list_permits,
+  read_permit: tool_read_permit,
+  query_matrix: tool_query_matrix,
+  note: tool_note,
+  read_scratchpad: tool_read_scratchpad,
+  done: tool_done,
+};
+
+const TOOL_SCHEMA = `Available tools (call by emitting JSON like: {"tool": "name", "args": {...}}):
+- list_permits(min_cost?: number, permit_type?: string) — top 5 by cost
+- read_permit(permit_id: string) — full permit fields
+- query_matrix(query: string, top_k?: number) — search KB
+- note(text: string) — append to scratchpad
+- read_scratchpad() — read your scratchpad
+- done(summary: string) — finish; pass final markdown analysis`;
+
+// ─── AGENT LOOP ───
+
+async function callAgent(messages: Array<{role: string; content: string}>): Promise<string> {
+  // think:false disables hidden reasoning so all generated tokens go to
+  // visible response. qwen3.5:latest defaults to thinking and silently
+  // burns the token budget otherwise.
+  const r = await fetch(`${SIDECAR}/generate`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      model: AGENT_MODEL,
+      prompt: messages.map(m => `${m.role.toUpperCase()}:\n${m.content}`).join("\n\n") + "\n\nASSISTANT:\n",
+      stream: false,
+      max_tokens: 1500,
+      think: false,
+    }),
+    signal: AbortSignal.timeout(180000),
+  });
+  if (!r.ok) throw new Error(`agent ${r.status}: ${(await r.text()).slice(0, 200)}`);
+  const j: any = await r.json();
+  return String(j.text ?? j.response ?? "").trim();
+}
+
+function extractToolCall(response: string): { tool: string; args: any } | null {
+  // Look for JSON block in the response
+  const fenced = response.match(/```(?:json)?\s*(\{[\s\S]+?\})\s*```/);
+  const candidate = fenced ? fenced[1] : (response.match(/\{[\s\S]*\}/)?.[0] ?? null);
+  if (!candidate) return null;
+  try {
+    const parsed = JSON.parse(candidate);
+    if (parsed.tool && typeof parsed.tool === "string") return { tool: parsed.tool, args: parsed.args ?? {} };
+  } catch { /* not JSON */ }
+  return null;
+}
+
+async function main() {
+  log(`session=${SESSION_ID} model=${AGENT_MODEL} max_steps=${MAX_STEPS}`);
+  // Reset workspace files for this session
+  for (const p of [SCRATCHPAD_PATH, TRACE_PATH, FINAL_PATH]) {
+    try { await Bun.write(p, ""); } catch { /* ignore */ }
+  }
+
+  const prd = await readFile(PRD_PATH, "utf8");
+  log(`loaded PRD (${prd.length} chars)`);
+  await emitObserverEvent({ event_kind: "agent_start", model: AGENT_MODEL });
+
+  // Pre-flight: pull prior accepted pathway traces for this task class
+  // and surface them as a "PROVEN APPROACHES" preamble. This closes the
+  // matrix loop — successful past runs now actively help the next agent.
+  let priorPlaybooks = "";
+  try {
+    const stateFile = Bun.file("/home/profit/lakehouse/data/_pathway_memory/state.json");
+    if (await stateFile.exists()) {
+      const state: any = JSON.parse(await stateFile.text());
+      const matched: any[] = [];
+      for (const traces of Object.values(state.pathways ?? {}) as any[][]) {
+        for (const t of traces) {
+          if (t.task_class === "chicago_permit_analysis" && t.final_verdict === "accepted" && !t.retired) {
+            matched.push(t);
+          }
+        }
+      }
+      matched.sort((a, b) => (b.created_at ?? "").localeCompare(a.created_at ?? ""));
+      if (matched.length > 0) {
+        const top = matched.slice(0, 2);
+        priorPlaybooks = "\n\n═══ 📖 PROVEN APPROACHES FROM PRIOR ACCEPTED RUNS ═══\n" +
+          top.map((t, i) =>
+            `[${i + 1}] pathway=${t.pathway_id?.slice(0, 12)} previously succeeded on ${t.file_path}\n` +
+            `Approach excerpt:\n${(t.reducer_summary ?? "").slice(0, 800)}`
+          ).join("\n\n") +
+          "\n═══ end proven approaches ═══\n\nUse these as REFERENCE for what worked. Don't copy verbatim, but follow the same workflow shape (plan → list → read → matrix → analyze → done).\n";
+        log(`📖 found ${matched.length} prior accepted pathway(s) for chicago_permit_analysis — top ${top.length} prepended to agent context`);
+      } else {
+        log(`📖 no prior accepted pathways for chicago_permit_analysis (this is the first run)`);
+      }
+    }
+  } catch (e: any) {
+    log(`📖 pathway preamble skipped: ${e.message}`);
+  }
+
+  const systemMsg = `You are an autonomous agent. Read the PRD below and follow its instructions exactly.
+
+${TOOL_SCHEMA}
+
+To call a tool, respond with ONLY a JSON object: {"tool": "<name>", "args": {...}}
+No markdown, no explanation around it. The harness will execute the tool and give you the result, then ask you what to do next.
+
+When you are completely finished, call done(summary="<your final markdown>").`;
+
+  const messages: Array<{role: string; content: string}> = [
+    { role: "system", content: systemMsg },
+    { role: "user", content: `PRD:\n\n${prd}${priorPlaybooks}\n\nNow respond. Remember: PLAN first via note() before executing.` },
+  ];
+
+  // Iter 3 surfaced: when the matrix returns real evidence, the agent
+  // gets analysis paralysis — keeps calling note() to refine instead of
+  // producing the final output. Guard: after MAX_CONSECUTIVE_NOTES
+  // note() calls in a row, harness injects a hard-stop user message
+  // telling the agent it MUST call done() next.
+  const MAX_CONSECUTIVE_NOTES = Number(process.env.AGENT_MAX_CONSECUTIVE_NOTES ?? 2);
+  let consecutiveNotes = 0;
+
+  let isDone = false;
+  for (let step = 1; step <= MAX_STEPS && !isDone; step++) {
+    log(`step ${step}/${MAX_STEPS} — calling agent...`);
+    const t0 = Date.now();
+    let response: string;
+    try {
+      response = await callAgent(messages);
+    } catch (e: any) {
+      log(`  ✗ agent error: ${e.message}`);
+      await trace({ step, kind: "error", error: e.message });
+      await emitObserverEvent({ event_kind: "agent_error", step, error: e.message });
+      break;
+    }
+    const ms = Date.now() - t0;
+    log(`  · agent responded ${response.length} chars in ${ms}ms`);
+    await trace({ step, kind: "agent_response", chars: response.length, latency_ms: ms, response: response.slice(0, 4000) });
+
+    const call = extractToolCall(response);
+    if (!call) {
+      log(`  ⚠ no tool call extracted from response — agent may be confused`);
+      await trace({ step, kind: "no_tool_call", preview: response.slice(0, 500) });
+      await emitObserverEvent({ event_kind: "agent_no_tool", step, preview: response.slice(0, 200) });
+      // Push the agent: tell it to call a tool
+      messages.push({ role: "assistant", content: response });
+      messages.push({ role: "user", content: `Your last response did not contain a valid tool call. Respond with ONLY a JSON object like {"tool": "note", "args": {"text": "..."}}. No prose around it.` });
+      continue;
+    }
+
+    log(`  → tool: ${call.tool}(${JSON.stringify(call.args).slice(0, 200)})`);
+    if (!TOOLS[call.tool]) {
+      const err = `unknown tool: ${call.tool}`;
+      log(`  ✗ ${err}`);
+      await trace({ step, kind: "tool_unknown", tool: call.tool });
+      await emitObserverEvent({ event_kind: "tool_unknown", step, tool: call.tool });
+      messages.push({ role: "assistant", content: response });
+      messages.push({ role: "user", content: `Tool "${call.tool}" does not exist. Available: ${Object.keys(TOOLS).join(", ")}. Try again.` });
+      continue;
+    }
+
+    const resStart = Date.now();
+    let result: string;
+    try {
+      result = await TOOLS[call.tool](call.args);
+    } catch (e: any) {
+      result = `TOOL ERROR: ${e.message}`;
+    }
+    const resMs = Date.now() - resStart;
+    log(`  ← ${result.slice(0, 200)}${result.length > 200 ? "..." : ""} (${resMs}ms)`);
+    await trace({ step, kind: "tool_call", tool: call.tool, args: call.args, result: result.slice(0, 4000), latency_ms: resMs });
+    await emitObserverEvent({ event_kind: "tool_call", step, tool: call.tool, result_chars: result.length });
+
+    if (call.tool === "done") {
+      isDone = true;
+      log(`  ✓ DONE`);
+      await emitObserverEvent({ event_kind: "agent_done", step });
+      break;
+    }
+
+    // Track consecutive note() calls; force done() if too many in a row.
+    if (call.tool === "note") consecutiveNotes++;
+    else consecutiveNotes = 0;
+
+    messages.push({ role: "assistant", content: response });
+    if (consecutiveNotes >= MAX_CONSECUTIVE_NOTES) {
+      log(`  ⚠ ${consecutiveNotes} consecutive note() calls — forcing done() next`);
+      await emitObserverEvent({ event_kind: "force_done_pressure", step, consecutive_notes: consecutiveNotes });
+      messages.push({ role: "user", content: `Tool result:\n${result}\n\nYou have called note() ${consecutiveNotes} times in a row without producing output. STOP NOTING. Call done(summary="<your final markdown>") NOW with whatever analysis you have. Do not call note() again. Respond with ONLY: {"tool": "done", "args": {"summary": "..."}}` });
+      consecutiveNotes = 0; // reset so we only push once per streak
+    } else {
+      messages.push({ role: "user", content: `Tool result:\n${result}\n\nWhat next?` });
+    }
+  }
+
+  if (!isDone) {
+    log(`✗ agent did not complete within ${MAX_STEPS} steps`);
+    await emitObserverEvent({ event_kind: "agent_max_steps", final_step: MAX_STEPS });
+    // Mem0: any partial trace this session inserted should be retired
+    // so future agents don't get a broken playbook in their preamble.
+    // We don't have a trace_uid for this session yet (insert happens
+    // on done); but if any prior trace has the same workflow shape as
+    // this session's tool sequence, retire it.
+    // For now, just log — actual retirement would happen if seal had run.
+    log(`  ⚠ no playbook seal will be performed for failed run`);
+  }
+
+  log(`session ${SESSION_ID} ended. Trace: ${TRACE_PATH}`);
+  if (existsSync(FINAL_PATH)) log(`Final output: ${FINAL_PATH}`);
+}
+
+mkdirSync("/home/profit/lakehouse/tests/agent_test", { recursive: true });
+main().catch(e => { console.error(`FATAL: ${e.message}`); process.exit(1); });

tests/architecture_smoke.ts

@@ -0,0 +1,228 @@
+// Architecture smoke test — exercises the lakehouse substrate against
+// the 500k-worker reference dataset. Proves the architecture's core
+// claims from docs/PRD.md §Shared requirements end-to-end in <60s,
+// without cloud calls.
+//
+// Each section tests ONE invariant. PASS or FAIL printed per section
+// with the actual measurement. A FAIL tells us exactly where the
+// architecture broke; the failure is the driver for the next commit.
+//
+// Run: bun run tests/architecture_smoke.ts
+//
+// Expected preconditions (checked at top):
+//   - gateway on :3100
+//   - sidecar on :3200 (for embed endpoint)
+//   - workers_500k dataset registered in catalog (500k rows)
+//   - workers_500k_v1 vector index built (50k chunks, 768d nomic)
+//   - primary bucket accessible
+
+const GATEWAY = process.env.GATEWAY_URL ?? "http://localhost:3100";
+const SIDECAR = process.env.SIDECAR_URL ?? "http://localhost:3200";
+
+type Check = {
+  name: string;
+  ok: boolean;
+  measure?: string;
+  err?: string;
+};
+const results: Check[] = [];
+
+function record(name: string, ok: boolean, measure?: string, err?: string) {
+  results.push({ name, ok, measure, err });
+  const marker = ok ? "✓" : "✗";
+  const detail = ok ? (measure ?? "") : (err ?? "");
+  console.log(`  ${marker} ${name.padEnd(50)} ${detail}`);
+}
+
+async function http<T = any>(path: string, init?: RequestInit): Promise<T> {
+  const r = await fetch(`${GATEWAY}${path}`, {
+    ...init,
+    headers: { "content-type": "application/json", ...(init?.headers ?? {}) },
+  });
+  if (!r.ok) throw new Error(`${path} ${r.status}: ${await r.text().catch(() => "?")}`);
+  return r.json() as Promise<T>;
+}
+
+async function main() {
+  const t0 = Date.now();
+  console.log(`\n━━━ Architecture smoke test ━━━`);
+  console.log(`Gateway: ${GATEWAY}`);
+  console.log(`Sidecar: ${SIDECAR}`);
+  console.log();
+
+  // ─── 1. Preconditions ─────────────────────────────────────────
+  console.log("【 1 · preconditions 】");
+  try {
+    const h = await fetch(`${GATEWAY}/health`);
+    record("gateway /health", h.ok, `HTTP ${h.status}`);
+  } catch (e) {
+    record("gateway /health", false, undefined, String(e));
+  }
+  try {
+    const h = await fetch(`${SIDECAR}/health`);
+    record("sidecar /health", h.ok, `HTTP ${h.status}`);
+  } catch (e) {
+    record("sidecar /health", false, undefined, String(e));
+  }
+
+  // ─── 2. Catalog — invariant: any dataset is queryable by name ──
+  console.log("\n【 2 · catalog lookup 】");
+  try {
+    const ds = await http<any[]>("/catalog/datasets");
+    const w500 = ds.find((d) => d.name === "workers_500k");
+    if (w500) {
+      record("workers_500k in catalog", true, `${w500.row_count ?? "?"} rows`);
+    } else {
+      record("workers_500k in catalog", false, undefined, "not found");
+    }
+  } catch (e) {
+    record("workers_500k in catalog", false, undefined, String(e));
+  }
+
+  // ─── 3. SQL at scale — PRD §shared: millions of rows, sub-second ──
+  console.log("\n【 3 · SQL at scale 】");
+  await timed("count(*) on workers_500k", async () => {
+    const j = await http<any>("/query/sql", {
+      method: "POST",
+      body: JSON.stringify({ sql: "SELECT COUNT(*) as n FROM workers_500k" }),
+    });
+    const n = j?.rows?.[0]?.n ?? 0;
+    if (n < 400_000) throw new Error(`only ${n} rows — expected ~500k`);
+    return `${n.toLocaleString()} rows`;
+  });
+  await timed("geo filter on workers_500k", async () => {
+    const j = await http<any>("/query/sql", {
+      method: "POST",
+      body: JSON.stringify({
+        sql: "SELECT COUNT(*) as n FROM workers_500k WHERE state = 'OH'",
+      }),
+    });
+    return `${(j?.rows?.[0]?.n ?? 0).toLocaleString()} Ohio workers`;
+  });
+
+  // ─── 4. Vector search — PRD §shared: AI embeddings over a profile index ──
+  // Gateway endpoint takes TEXT + embeds internally (uses the sidecar's
+  // /embed under the hood). We don't pre-embed on the client side.
+  console.log("\n【 4 · vector search 】");
+  await timed("vector /vectors/search on workers_500k_v1", async () => {
+    const j = await http<any>("/vectors/search", {
+      method: "POST",
+      body: JSON.stringify({
+        index_name: "workers_500k_v1",
+        query: "experienced welder for industrial work",
+        top_k: 10,
+      }),
+    });
+    const results = j?.results ?? [];
+    if (!Array.isArray(results) || results.length === 0) {
+      throw new Error(`no results: ${JSON.stringify(j).slice(0, 200)}`);
+    }
+    return `${results.length} results`;
+  });
+
+  // ─── 5. Hybrid SQL+vector — PRD §shared: scoped view via filter ──
+  console.log("\n【 5 · hybrid search 】");
+  await timed("hybrid: 'welder in Toledo, OH', SQL-filtered", async () => {
+    const j = await http<any>("/vectors/hybrid", {
+      method: "POST",
+      body: JSON.stringify({
+        index_name: "workers_500k_v1",
+        question: "experienced welder",
+        sql_filter: "state = 'OH' AND city = 'Toledo'",
+        k: 10,
+      }),
+    });
+    const results = j?.results ?? j?.hits ?? [];
+    if (!Array.isArray(results)) {
+      throw new Error(`malformed response: ${JSON.stringify(j).slice(0, 200)}`);
+    }
+    return `${results.length} Toledo-OH hits`;
+  });
+
+  // ─── 6. Playbook memory — PRD §shared: trials as first-class data ──
+  console.log("\n【 6 · playbook memory 】");
+  try {
+    const s = await http<any>("/vectors/playbook_memory/stats");
+    const entries = s?.entries_count ?? s?.count ?? 0;
+    record("playbook_memory populated", true, `${entries} entries`);
+  } catch (e) {
+    record("playbook_memory populated", false, undefined, String(e));
+  }
+
+  // ─── 7. Pathway memory (ADR-021) — the compounding layer ──
+  console.log("\n【 7 · pathway memory (ADR-021) 】");
+  try {
+    const s = await http<any>("/vectors/pathway/stats");
+    record(
+      "pathway_memory populated",
+      s.total_pathways > 0,
+      `${s.total_pathways} traces, ${s.retired} retired, ${s.with_audit_pass} audit-pass`,
+    );
+    // Probe the bug_fingerprints endpoint we rely on for preamble
+    const bf = await http<any>("/vectors/pathway/bug_fingerprints", {
+      method: "POST",
+      body: JSON.stringify({
+        task_class: "scrum_review",
+        file_path: "crates/queryd/src/delta.rs",
+        signal_class: null,
+        limit: 5,
+      }),
+    });
+    record(
+      "bug_fingerprints endpoint",
+      Array.isArray(bf.fingerprints),
+      `${bf.fingerprints?.length ?? 0} patterns`,
+    );
+  } catch (e) {
+    record("pathway_memory stats", false, undefined, String(e));
+  }
+
+  // ─── 8. Truth gate — SQL safety on query path ──
+  console.log("\n【 8 · truth gate (SQL safety) 】");
+  try {
+    const r = await fetch(`${GATEWAY}/query/sql`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ sql: "DROP TABLE workers_500k" }),
+    });
+    record(
+      "destructive SQL blocked on /query/sql",
+      r.status === 403,
+      `HTTP ${r.status}`,
+      r.status === 403 ? undefined : `expected 403, got ${r.status}`,
+    );
+  } catch (e) {
+    record("destructive SQL blocked on /query/sql", false, undefined, String(e));
+  }
+
+  // ─── Summary ─────────────────────────────────────────────────
+  const elapsed = Date.now() - t0;
+  const passed = results.filter((r) => r.ok).length;
+  const failed = results.filter((r) => !r.ok).length;
+  console.log();
+  console.log(`━━━ Summary ━━━`);
+  console.log(`  ${passed}/${passed + failed} passed · ${elapsed}ms elapsed`);
+  if (failed > 0) {
+    console.log();
+    console.log(`FAILED checks — these are the next things to fix:`);
+    for (const r of results) {
+      if (!r.ok) console.log(`  ✗ ${r.name}: ${r.err ?? "?"}`);
+    }
+    process.exit(1);
+  }
+  console.log(`✓ architecture smoke passed`);
+}
+
+async function timed(name: string, fn: () => Promise<string | undefined>) {
+  const t0 = Date.now();
+  try {
+    const detail = await fn();
+    const ms = Date.now() - t0;
+    record(name, true, `${ms}ms ${detail ? `· ${detail}` : ""}`);
+  } catch (e) {
+    const ms = Date.now() - t0;
+    record(name, false, `${ms}ms`, String(e));
+  }
+}
+
+await main();

tests/multi-agent/agent.ts

@@ -394,11 +394,15 @@ export async function generate(model: string, prompt: string, opts: {
   return text;
 }
 
-// Cloud generate — hits Ollama Cloud directly with the bearer key. Same
+// Cloud generate — routes through the lakehouse gateway's /v1/chat
-// /api/generate shape as local Ollama; `thinking` field (for gpt-oss:Nb)
+// with provider="ollama_cloud". Phase 44 migration (2026-04-24): was
-// is discarded, only `response` is returned. Caller should budget
+// hitting OLLAMA_CLOUD_URL/api/generate directly with a bearer key,
-// num_predict ≥ 400 so thinking-model reasoning has room before the
+// bypassing the gateway's usage tracking + audit path. Now every call
-// visible response starts.
+// flows through /v1/chat so /v1/usage accounts for it. Gateway holds
+// the OLLAMA_CLOUD_KEY; caller no longer needs it in env.
+//
+// Thinking-model budget note: num_predict ≥ 400 still matters, just
+// expressed via max_tokens on the /v1/chat request.
 export async function generateCloud(model: string, prompt: string, opts: {
   max_tokens?: number;
   temperature?: number;
@@ -406,41 +410,35 @@ export async function generateCloud(model: string, prompt: string, opts: {
   bypass_budget?: boolean;
   think?: boolean;
 } = {}): Promise<string> {
-  if (!OLLAMA_CLOUD_KEY) {
-    throw new Error("OLLAMA_CLOUD_KEY not set; cannot reach Ollama Cloud");
-  }
   assertContextBudget(model, prompt, {
     system: opts.system,
     max_tokens: opts.max_tokens,
     bypass: opts.bypass_budget,
   });
+  const messages: Array<{ role: string; content: string }> = [];
+  if (opts.system) messages.push({ role: "system", content: opts.system });
+  messages.push({ role: "user", content: prompt });
   const body: Record<string, any> = {
     model,
-    prompt,
+    messages,
-    stream: false,
+    provider: "ollama_cloud",
-    options: {
     temperature: opts.temperature ?? 0.3,
-      num_predict: Math.max(opts.max_tokens ?? 800, 400),
+    max_tokens: Math.max(opts.max_tokens ?? 800, 400),
-    },
   };
-  if (opts.system) body.system = opts.system;
   if (opts.think !== undefined) body.think = opts.think;
-  const resp = await fetch(`${OLLAMA_CLOUD_URL}/api/generate`, {
+  const resp = await fetch(`${GATEWAY}/v1/chat`, {
     method: "POST",
-    headers: {
+    headers: { "Content-Type": "application/json" },
-      "Authorization": `Bearer ${OLLAMA_CLOUD_KEY}`,
-      "Content-Type": "application/json",
-    },
     body: JSON.stringify(body),
   });
   if (!resp.ok) {
-    throw new Error(`Ollama Cloud ${resp.status}: ${await resp.text().catch(() => "?")}`);
+    throw new Error(`gateway /v1/chat ${resp.status}: ${await resp.text().catch(() => "?")}`);
   }
   const data: any = await resp.json();
-  const text = typeof data.response === "string" ? data.response : "";
+  const text = data?.choices?.[0]?.message?.content ?? "";
   // Same non-throw policy as local generate() — empty text is a valid
   // signal that thinking ate the budget. Let generateContinuable retry.
-  return text;
+  return typeof text === "string" ? text : "";
 }
 
 // --- Prompt construction ---

tests/real-world/autonomous_loop.ts

@@ -0,0 +1,214 @@
+#!/usr/bin/env bun
+// Autonomous scrum loop — wraps scrum_master_pipeline.ts + scrum_applier.ts
+// in a goal-driven retry loop. Observer is POSTed an iteration summary at
+// every boundary so it can build meta-commentary outside the loop's epistemic
+// scope.
+//
+// Usage:
+//   LOOP_TARGETS="crates/a/src/x.rs,crates/b/src/y.rs" \
+//   LOOP_MAX_ITERS=5 \
+//   LOOP_PUSH=1 \
+//   bun run tests/real-world/autonomous_loop.ts
+//
+// Stop conditions: max_iters reached OR 2 consecutive iters with 0 commits.
+
+import { spawn } from "node:child_process";
+import { appendFile, readFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+
+const REPO = "/home/profit/lakehouse";
+const OBSERVER = process.env.LOOP_OBSERVER ?? "http://localhost:3800";
+const BRANCH = process.env.LOOP_BRANCH ?? "scrum/auto-apply-19814";
+const MAX_ITERS = Number(process.env.LOOP_MAX_ITERS ?? 3);
+const PUSH = process.env.LOOP_PUSH === "1";
+const MIN_CONF = process.env.LOOP_MIN_CONF ?? "85";
+// Optional override — when unset, let scrum_applier.ts use ITS default
+// (currently x-ai/grok-4.1-fast on openrouter). The prior hardcoded
+// qwen3-coder:480b default was clobbering the applier's own default
+// and forcing every iter to hit the throttled ollama_cloud account.
+const APPLIER_MODEL = process.env.LOOP_APPLIER_MODEL;
+const APPLIER_PROVIDER = process.env.LOOP_APPLIER_PROVIDER;
+const TARGETS = (process.env.LOOP_TARGETS ?? "crates/queryd/src/service.rs,crates/gateway/src/main.rs,crates/gateway/src/v1/mod.rs")
+  .split(",").map(s => s.trim()).filter(Boolean);
+
+const FORENSIC = process.env.LH_SCRUM_FORENSIC ?? `${REPO}/docs/SCRUM_FORENSIC_PROMPT.md`;
+const PROPOSAL = process.env.LH_SCRUM_PROPOSAL ?? `${REPO}/docs/SCRUM_FIX_WAVE.md`;
+
+const LOOP_ID = `loop_${Date.now().toString(36)}`;
+const JOURNAL = `${REPO}/data/_kb/autonomous_loops.jsonl`;
+
+interface IterResult {
+  iter: number;
+  scrum_reviews_added: number;
+  applier_outcomes: Record<string, number>;
+  commits_landed: number;
+  commit_shas: string[];
+  build_status: "green" | "red" | "unknown";
+  duration_ms: number;
+}
+
+function log(msg: string) {
+  const ts = new Date().toISOString().slice(11, 19);
+  console.log(`[loop ${LOOP_ID} ${ts}] ${msg}`);
+}
+
+function runCmd(cmd: string, args: string[], env: Record<string, string> = {}): Promise<{ code: number; stdout: string; stderr: string }> {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args, { cwd: REPO, env: { ...process.env, ...env } });
+    let stdout = "", stderr = "";
+    child.stdout.on("data", (d) => { stdout += d; process.stdout.write(d); });
+    child.stderr.on("data", (d) => { stderr += d; process.stderr.write(d); });
+    child.on("close", (code) => resolve({ code: code ?? -1, stdout, stderr }));
+  });
+}
+
+async function countLines(path: string): Promise<number> {
+  if (!existsSync(path)) return 0;
+  const text = await readFile(path, "utf8");
+  return text.split("\n").filter(Boolean).length;
+}
+
+async function gitHeadSha(): Promise<string> {
+  const r = await runCmd("git", ["rev-parse", "HEAD"]);
+  return r.stdout.trim();
+}
+
+async function commitsSince(baseSha: string): Promise<string[]> {
+  const r = await runCmd("git", ["log", "--oneline", `${baseSha}..HEAD`]);
+  return r.stdout.trim().split("\n").filter(Boolean);
+}
+
+async function cargoCheckGreen(): Promise<boolean> {
+  log("cargo check --workspace …");
+  const r = await runCmd("cargo", ["check", "--workspace", "--quiet"]);
+  return r.code === 0;
+}
+
+async function postObserver(payload: object) {
+  try {
+    const r = await fetch(`${OBSERVER}/event`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(5000),
+    });
+    if (!r.ok) log(`observer POST returned ${r.status}`);
+  } catch (e: any) {
+    log(`observer POST failed: ${e.message}`);
+  }
+}
+
+async function runIter(iter: number, baseSha: string): Promise<IterResult> {
+  const t0 = Date.now();
+  log(`══ iter ${iter} start (base ${baseSha.slice(0, 8)}) targets=${TARGETS.length}`);
+
+  const reviewsBefore = await countLines(`${REPO}/data/_kb/scrum_reviews.jsonl`);
+  const applyBefore = await countLines(`${REPO}/data/_kb/auto_apply.jsonl`);
+
+  log(`scrum_master_pipeline.ts → ${TARGETS.length} files`);
+  await runCmd("bun", ["run", "tests/real-world/scrum_master_pipeline.ts"], {
+    LH_SCRUM_FILES: TARGETS.join(","),
+    LH_SCRUM_FORENSIC: FORENSIC,
+    LH_SCRUM_PROPOSAL: PROPOSAL,
+  });
+
+  log(`scrum_applier.ts COMMIT=1 MIN_CONF=${MIN_CONF} files=${TARGETS.length}`);
+  // Only forward model/provider when explicitly overridden — otherwise
+  // let scrum_applier.ts use its own defaults (Grok 4.1 fast on openrouter).
+  const applierEnv: Record<string, string> = {
+    LH_APPLIER_COMMIT: "1",
+    LH_APPLIER_MIN_CONF: MIN_CONF,
+    LH_APPLIER_MAX_FILES: String(TARGETS.length),
+    LH_APPLIER_BRANCH: BRANCH,
+    // Constrain applier to THIS iter's targets so it patches what we
+    // just reviewed instead of the highest-confidence file from history.
+    LH_APPLIER_FILES: TARGETS.join(","),
+  };
+  if (APPLIER_MODEL) applierEnv.LH_APPLIER_MODEL = APPLIER_MODEL;
+  if (APPLIER_PROVIDER) applierEnv.LH_APPLIER_PROVIDER = APPLIER_PROVIDER;
+  await runCmd("bun", ["run", "tests/real-world/scrum_applier.ts"], applierEnv);
+
+  const reviewsAfter = await countLines(`${REPO}/data/_kb/scrum_reviews.jsonl`);
+  const applyAfterText = existsSync(`${REPO}/data/_kb/auto_apply.jsonl`)
+    ? await readFile(`${REPO}/data/_kb/auto_apply.jsonl`, "utf8")
+    : "";
+  const applyRows = applyAfterText.split("\n").filter(Boolean).slice(applyBefore);
+  const outcomes: Record<string, number> = {};
+  for (const line of applyRows) {
+    try {
+      const o = JSON.parse(line);
+      outcomes[o.action ?? "?"] = (outcomes[o.action ?? "?"] ?? 0) + 1;
+    } catch { /* skip malformed */ }
+  }
+
+  const commitShas = await commitsSince(baseSha);
+  const buildStatus = commitShas.length > 0 ? (await cargoCheckGreen() ? "green" : "red") : "unknown";
+
+  const result: IterResult = {
+    iter,
+    scrum_reviews_added: reviewsAfter - reviewsBefore,
+    applier_outcomes: outcomes,
+    commits_landed: commitShas.length,
+    commit_shas: commitShas.map(s => s.split(" ")[0]),
+    build_status: buildStatus,
+    duration_ms: Date.now() - t0,
+  };
+
+  log(`iter ${iter} done — reviews+${result.scrum_reviews_added} commits=${result.commits_landed} build=${buildStatus} (${(result.duration_ms / 1000).toFixed(1)}s)`);
+
+  await postObserver({
+    source: "autonomous_loop",
+    loop_id: LOOP_ID,
+    event_kind: "iteration_complete",
+    iter,
+    targets: TARGETS,
+    success: buildStatus !== "red",
+    scrum_reviews_added: result.scrum_reviews_added,
+    applier_outcomes: result.applier_outcomes,
+    commits_landed: result.commits_landed,
+    commit_shas: result.commit_shas,
+    build_status: buildStatus,
+    duration_ms: result.duration_ms,
+    ts: new Date().toISOString(),
+  });
+
+  await appendFile(JOURNAL, JSON.stringify({ loop_id: LOOP_ID, ...result, ts: new Date().toISOString() }) + "\n");
+
+  return result;
+}
+
+async function main() {
+  log(`autonomous loop starting · branch=${BRANCH} max_iters=${MAX_ITERS} push=${PUSH}`);
+  log(`targets: ${TARGETS.join(", ")}`);
+
+  const branchR = await runCmd("git", ["branch", "--show-current"]);
+  if (branchR.stdout.trim() !== BRANCH) {
+    log(`ERROR: on branch ${branchR.stdout.trim()}, expected ${BRANCH}. Refusing to run.`);
+    process.exit(1);
+  }
+
+  let consecutiveZero = 0;
+  for (let iter = 1; iter <= MAX_ITERS; iter++) {
+    const baseSha = await gitHeadSha();
+    const result = await runIter(iter, baseSha);
+
+    if (PUSH && result.commits_landed > 0) {
+      log(`git push origin ${BRANCH}`);
+      const pushR = await runCmd("git", ["push", "origin", BRANCH]);
+      if (pushR.code !== 0) log(`push failed (continuing): ${pushR.stderr.slice(0, 200)}`);
+    }
+
+    consecutiveZero = result.commits_landed === 0 ? consecutiveZero + 1 : 0;
+    if (consecutiveZero >= 2) {
+      log(`STOP: 2 consecutive iters with 0 commits. Loop converged or stuck.`);
+      break;
+    }
+  }
+
+  log(`loop ${LOOP_ID} complete. Journal: ${JOURNAL}`);
+}
+
+main().catch((e) => {
+  log(`FATAL: ${e.message}`);
+  process.exit(1);
+});

tests/real-world/scrum_applier.ts

@@ -0,0 +1,487 @@
+// scrum_applier.ts — the auto-apply pipeline.
+//
+// Turns the scrum master's signal into real commits. Reads
+// data/_kb/scrum_reviews.jsonl, filters to rows where the scrum's
+// own confidence is high enough to trust auto-apply (gradient_tier
+// auto OR confidence_avg ≥ 90), asks a patch-emitting model to
+// produce concrete old_string/new_string pairs, applies them via
+// text replacement, runs `cargo check` after each, commits on green
+// and reverts on red.
+//
+// Runs on its own branch (never on main). Every action is recorded
+// in data/_kb/auto_apply.jsonl so the auditor and future iterations
+// can see what landed and what reverted.
+//
+// Usage:
+//   bun run tests/real-world/scrum_applier.ts                 # dry-run, print only
+//   LH_APPLIER_COMMIT=1 bun run tests/real-world/scrum_applier.ts   # actually apply
+//
+// Env:
+//   LH_APPLIER_BRANCH  — branch name (default: "scrum/auto-apply-${Date.now()}")
+//   LH_APPLIER_MIN_CONF — minimum confidence_avg, default 90
+//   LH_APPLIER_MAX_FILES — cap on files per run (default 5, keeps diffs reviewable)
+//   LH_APPLIER_COMMIT — "1" to actually commit; otherwise dry-run
+//   LH_APPLIER_MODEL  — patch-emitting model (default: kimi-k2:1t)
+
+import { readFile, writeFile, appendFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { spawn } from "node:child_process";
+
+const REPO = "/home/profit/lakehouse";
+const GATEWAY = "http://localhost:3100";
+const SCRUM_REVIEWS = `${REPO}/data/_kb/scrum_reviews.jsonl`;
+const AUDIT_LOG = `${REPO}/data/_kb/auto_apply.jsonl`;
+
+const MIN_CONF = Number(process.env.LH_APPLIER_MIN_CONF ?? 90);
+const MAX_FILES = Number(process.env.LH_APPLIER_MAX_FILES ?? 5);
+const COMMIT = process.env.LH_APPLIER_COMMIT === "1";
+// LH_APPLIER_FILES — comma-separated repo-relative paths. When set,
+// constrains eligible reviews to ONLY those files. Used by the autonomous
+// loop so the applier patches what scrum just reviewed in this iter,
+// instead of pulling the highest-confidence file from global review history.
+const TARGET_FILES = (process.env.LH_APPLIER_FILES ?? "")
+  .split(",").map(s => s.trim()).filter(Boolean);
+// Default patch-emitter model — qwen3-coder:480b is the coding specialist
+// in the scrum ladder (rung 2). Swapped in from kimi-k2:1t after 2026-04-24
+// data showed kimi-k2:1t produces architectural patches that cascade across
+// the file (rename a field → 20 broken call sites). qwen3-coder is tuned
+// for targeted code changes and tends to stay within the mechanical-patch
+// constraint the prompt asks for. LLM Team's /api/run?mode=patch would be
+// the ideal choice but that mode isn't registered in llm_team_ui.py yet.
+// Default patch emitter swapped to OpenRouter Grok 4.1 fast (2026-04-25)
+// after observing the prior default (ollama_cloud::qwen3-coder:480b) sit
+// at 429 throttle and never produce patches. Grok 4.1 fast: $0.20/$0.50
+// per M, 2M ctx, proven to emit precise structured patches in observer
+// hand-review tests. Override with LH_APPLIER_MODEL + LH_APPLIER_PROVIDER.
+const MODEL = process.env.LH_APPLIER_MODEL ?? "x-ai/grok-4.1-fast";
+const PROVIDER = (process.env.LH_APPLIER_PROVIDER ?? "openrouter") as "ollama_cloud" | "openrouter" | "ollama";
+const BRANCH = process.env.LH_APPLIER_BRANCH ?? `scrum/auto-apply-${Date.now().toString(36)}`;
+
+// Deny-list — anything whose path starts with one of these is skipped
+// regardless of how confident the scrum is. Config / systemd / docs /
+// auditor itself are off limits for auto-apply; they need a human.
+const DENY_PREFIXES = [
+  "config/",
+  "ops/",
+  "auditor/",
+  "docs/",
+  "data/",
+  "/etc/",
+  "mcp-server/",
+  "ui/",
+  "sidecar/",
+  "scripts/",
+];
+
+function log(msg: string) { console.log(`[applier] ${msg}`); }
+
+async function sh(cmd: string[], cwd = REPO): Promise<{ stdout: string; stderr: string; code: number }> {
+  return new Promise((resolve) => {
+    const p = spawn(cmd[0], cmd.slice(1), { cwd, stdio: ["ignore", "pipe", "pipe"] });
+    let out = ""; let err = "";
+    p.stdout.on("data", (d) => { out += d.toString(); });
+    p.stderr.on("data", (d) => { err += d.toString(); });
+    p.on("close", (code) => resolve({ stdout: out, stderr: err, code: code ?? 1 }));
+  });
+}
+
+async function auditLog(row: Record<string, any>) {
+  const line = JSON.stringify({ ...row, ts: new Date().toISOString() }) + "\n";
+  await appendFile(AUDIT_LOG, line);
+}
+
+async function chat(opts: {
+  provider: "ollama_cloud" | "openrouter" | "ollama";
+  model: string;
+  prompt: string;
+  max_tokens?: number;
+}): Promise<{ content: string; error?: string }> {
+  try {
+    const r = await fetch(`${GATEWAY}/v1/chat`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        provider: opts.provider,
+        model: opts.model,
+        messages: [{ role: "user", content: opts.prompt }],
+        max_tokens: opts.max_tokens ?? 1500,
+        temperature: 0.1,
+      }),
+      signal: AbortSignal.timeout(180000),
+    });
+    if (!r.ok) return { content: "", error: `${r.status}: ${(await r.text()).slice(0, 300)}` };
+    const j: any = await r.json();
+    return { content: j.choices?.[0]?.message?.content ?? "" };
+  } catch (e) {
+    return { content: "", error: String(e) };
+  }
+}
+
+interface ScrumReview {
+  file: string;
+  reviewed_at: string;
+  accepted_model: string;
+  suggestions_preview: string;
+  confidences_per_finding?: number[];
+  confidence_avg?: number | null;
+  confidence_min?: number | null;
+  gradient_tier?: string;
+  gradient_tier_avg?: string;
+  verdict?: string;
+  critical_failures_count?: number;
+  schema_version?: number;
+}
+
+async function loadLatestReviews(): Promise<Map<string, ScrumReview>> {
+  // Map of file → latest review for that file. Ordered by reviewed_at.
+  if (!existsSync(SCRUM_REVIEWS)) return new Map();
+  const text = await readFile(SCRUM_REVIEWS, "utf8");
+  const rows: ScrumReview[] = text.split("\n").filter(Boolean).map(l => {
+    try { return JSON.parse(l); } catch { return null; }
+  }).filter((r): r is ScrumReview => r !== null);
+  // Keep the LATEST review per file.
+  const latest = new Map<string, ScrumReview>();
+  for (const r of rows) {
+    if (!r.file) continue;
+    const prev = latest.get(r.file);
+    if (!prev || (r.reviewed_at > prev.reviewed_at)) latest.set(r.file, r);
+  }
+  return latest;
+}
+
+function passesConfidenceGate(r: ScrumReview): boolean {
+  const avg = r.confidence_avg ?? 0;
+  const min = r.confidence_min ?? 0;
+  // Must be auto or dry_run tier AND confidence_min ≥ MIN_CONF.
+  // min is the conservative tier-lower-bound (one weak finding drags
+  // the whole file to "simulation" or "block" tier).
+  if (r.gradient_tier === "block" || r.gradient_tier === "simulation") return false;
+  return avg >= MIN_CONF && min >= 70;
+}
+
+function passesDenyList(file: string): boolean {
+  return !DENY_PREFIXES.some((p) => file.startsWith(p) || file === p.replace(/\/$/, ""));
+}
+
+interface Patch {
+  file: string;
+  old_string: string;
+  new_string: string;
+  rationale: string;
+  confidence: number;
+}
+
+async function requestPatches(file: string, source: string, review: string): Promise<Patch[]> {
+  const prompt = `You previously produced this review of ${file}:
+
+─── REVIEW ───
+${review}
+─── END REVIEW ───
+
+The review is high-confidence and the file is eligible for auto-apply. Produce CONCRETE PATCHES as JSON so they can be applied via string replacement.
+
+HARD CONSTRAINTS (violations → patch rejected):
+  1. Output ONE JSON object with a "patches" array. NO prose, no markdown fences.
+  2. Each patch is {"old_string": "...", "new_string": "...", "rationale": "short", "confidence": 0-100}.
+  3. "old_string" MUST appear EXACTLY ONCE in the file (verbatim, including whitespace + trailing newlines).
+  4. Max diff size: 6 lines changed per patch (NOT 20). If the change needs >6 lines, emit {"patches": []} — too risky for auto-apply.
+  5. Mechanical-only (the list — nothing else):
+       (a) remove a #[allow(dead_code)] marker on a function now wired elsewhere
+       (b) add a missing 'use' import statement
+       (c) add a single field to a struct (no renames)
+       (d) flip a single boolean/match-arm that doesn't cascade
+       (e) add a fire-and-forget log or tracing call
+  6. FORBIDDEN (automatic reject):
+       • struct field RENAMES (break all call sites)
+       • function signature changes
+       • new modules, new traits, new dependencies
+       • any change that requires editing another file to keep it compiling
+       • adding a 'use' import whose symbol is NOT referenced in the same patch
+         or already referenced in the surrounding file (no "import and hope")
+       • rationale that describes functionality not present in the diff
+         (if rationale says "destructive SQL filter", the diff must contain SQL/filter tokens)
+  7. Each "new_string" MUST compile in isolation with the same surrounding code.
+  8. Max 2 patches per file — quality over quantity.
+  9. The "rationale" field MUST use vocabulary that appears in the new_string.
+     Generic rationales like "improve code" or "add missing feature" are rejected.
+  10. If you cannot produce at least one high-confidence mechanical patch under these constraints, output {"patches": []}. Don't guess.
+
+─── SOURCE (${source.length} bytes) ───
+${source.slice(0, 14000)}
+─── END SOURCE ───
+
+Emit ONLY the JSON object.`;
+
+  const r = await chat({ provider: PROVIDER, model: MODEL, prompt, max_tokens: 2500 });
+  if (r.error || !r.content) return [];
+
+  // Strip markdown fences if model wrapped the JSON.
+  let raw = r.content.trim();
+  const fenceStart = raw.match(/^```(?:json)?\s*/);
+  if (fenceStart) raw = raw.slice(fenceStart[0].length);
+  if (raw.endsWith("```")) raw = raw.slice(0, -3).trim();
+  // Find first { and last } to extract JSON block if there's prose.
+  const first = raw.indexOf("{");
+  const last = raw.lastIndexOf("}");
+  if (first >= 0 && last > first) raw = raw.slice(first, last + 1);
+
+  try {
+    const obj = JSON.parse(raw);
+    const patches: Patch[] = (obj.patches ?? []).filter((p: any) =>
+      typeof p?.old_string === "string" &&
+      typeof p?.new_string === "string" &&
+      p.old_string !== p.new_string &&
+      p.old_string.length > 0 &&
+      typeof p?.confidence === "number"
+    ).map((p: any) => ({
+      file,
+      old_string: p.old_string,
+      new_string: p.new_string,
+      rationale: String(p.rationale ?? ""),
+      confidence: p.confidence,
+    }));
+    return patches;
+  } catch (e) {
+    log(`  ${file}: patch JSON parse failed — ${String(e).slice(0, 100)}`);
+    return [];
+  }
+}
+
+async function applyPatches(file: string, patches: Patch[]): Promise<{ applied: number; rejected: Array<{patch: Patch; reason: string}> }> {
+  const full = `${REPO}/${file}`;
+  let source = await readFile(full, "utf8");
+  const rejected: Array<{patch: Patch; reason: string}> = [];
+  let applied = 0;
+  for (const p of patches) {
+    // Confidence gate at the individual-patch level.
+    if (p.confidence < MIN_CONF) { rejected.push({patch: p, reason: `confidence ${p.confidence} < ${MIN_CONF}`}); continue; }
+    // Uniqueness gate.
+    const occurrences = source.split(p.old_string).length - 1;
+    if (occurrences === 0) { rejected.push({patch: p, reason: "old_string not found"}); continue; }
+    if (occurrences > 1) { rejected.push({patch: p, reason: `old_string appears ${occurrences}× (not unique)`}); continue; }
+    // Size gate — no patch touches > 6 lines (diff discipline; matches prompt).
+    // Raised from 20 to 6 after 2026-04-24 data showed most 10-20 line patches
+    // cascaded and broke the build. Mechanical changes genuinely fit in 6 lines.
+    const oldLines = p.old_string.split("\n").length;
+    const newLines = p.new_string.split("\n").length;
+    if (Math.max(oldLines, newLines) > 6) { rejected.push({patch: p, reason: `patch too large (${Math.max(oldLines,newLines)} lines, max 6)`}); continue; }
+    source = source.replace(p.old_string, p.new_string);
+    applied++;
+  }
+  if (applied > 0) await writeFile(full, source);
+  return { applied, rejected };
+}
+
+async function cargoCheck(): Promise<{ green: boolean; warnings: number }> {
+  const r = await sh(["cargo", "check", "--workspace"]);
+  // Count warnings: cargo emits "warning:" lines on stderr (and sometimes stdout).
+  // We count unique "warning: <msg>" opening lines, not continuation context.
+  const combined = r.stdout + r.stderr;
+  const warnings = (combined.match(/^warning: /gm) || []).length;
+  return { green: r.code === 0, warnings };
+}
+
+async function gitCommit(file: string, patches: Patch[]): Promise<boolean> {
+  if (!COMMIT) { log(`  (dry-run) would commit ${file}`); return true; }
+  const addR = await sh(["git", "add", file]);
+  if (addR.code !== 0) { log(`  git add failed: ${addR.stderr.slice(0, 200)}`); return false; }
+  const msg = `auto-apply: ${patches.length} high-confidence fix${patches.length === 1 ? "" : "es"} in ${file}\n\n${patches.map(p => `- ${p.rationale} (conf ${p.confidence}%)`).join("\n")}\n\n🤖 scrum_applier.ts`;
+  const commitR = await sh(["git", "commit", "-m", msg]);
+  if (commitR.code !== 0) { log(`  git commit failed: ${commitR.stderr.slice(0, 200)}`); return false; }
+  log(`  ✓ committed ${file}`);
+  return true;
+}
+
+async function revertFile(file: string): Promise<void> {
+  await sh(["git", "checkout", "--", file]);
+}
+
+// Cheap, conservative check that the rationale actually describes the
+// diff. Tokenizes the new_string (what's actually being added), drops
+// Rust keywords + common filler, and requires the rationale to share
+// at least one meaningful token. Catches rationale-diff divergence like
+// "Add destructive SQL filter" for a diff that only adds `use tracing;`.
+const STOPWORDS = new Set([
+  "use", "let", "pub", "mut", "fn", "self", "the", "a", "and", "or", "in",
+  "to", "for", "of", "on", "at", "add", "added", "adds", "new", "is",
+  "as", "if", "else", "match", "return", "crate", "mod", "struct",
+  "trait", "impl", "this", "that", "with", "from", "into", "by", "be",
+  "it", "its", "not", "one", "two", "line", "lines",
+]);
+function tokenize(s: string): Set<string> {
+  const out = new Set<string>();
+  for (const raw of s.toLowerCase().split(/[^a-z0-9_]+/)) {
+    if (raw.length >= 4 && !STOPWORDS.has(raw)) out.add(raw);
+  }
+  return out;
+}
+function rationaleMatchesDiff(p: Patch): boolean {
+  // Only check tokens introduced by the patch (what's in new but not old).
+  const newToks = tokenize(p.new_string);
+  const oldToks = tokenize(p.old_string);
+  const added = new Set([...newToks].filter((t) => !oldToks.has(t)));
+  if (added.size === 0) return true;  // pure deletion — rationale unconstrained
+  const rationaleToks = tokenize(p.rationale);
+  for (const t of added) if (rationaleToks.has(t)) return true;
+  return false;
+}
+
+async function main() {
+  log(`starting · min_conf=${MIN_CONF} max_files=${MAX_FILES} model=${MODEL} commit=${COMMIT}`);
+
+  if (COMMIT) {
+    const headR = await sh(["git", "rev-parse", "--abbrev-ref", "HEAD"]);
+    const currentBranch = headR.stdout.trim();
+    if (currentBranch === "main") {
+      log(`refusing to run on main — create a branch first or set LH_APPLIER_BRANCH`);
+      const coR = await sh(["git", "checkout", "-b", BRANCH]);
+      if (coR.code !== 0) { log(`could not create branch ${BRANCH}: ${coR.stderr.slice(0, 200)}`); process.exit(1); }
+      log(`working branch: ${BRANCH}`);
+    } else {
+      log(`working branch: ${currentBranch}`);
+    }
+  }
+
+  const reviews = await loadLatestReviews();
+  log(`loaded ${reviews.size} latest reviews`);
+
+  const eligible = [...reviews.values()].filter(r =>
+    passesConfidenceGate(r) && passesDenyList(r.file) &&
+    (TARGET_FILES.length === 0 || TARGET_FILES.includes(r.file))
+  ).sort((a, b) => (b.confidence_avg ?? 0) - (a.confidence_avg ?? 0));
+
+  if (TARGET_FILES.length > 0) {
+    log(`LH_APPLIER_FILES set — constrained to ${TARGET_FILES.length} target file(s): ${TARGET_FILES.join(", ")}`);
+  }
+  log(`${eligible.length} pass confidence gate + deny-list${TARGET_FILES.length > 0 ? " + target filter" : ""}`);
+  log(`taking top ${Math.min(MAX_FILES, eligible.length)} by confidence`);
+
+  // Establish pre-run warning baseline so post-patch cargo check can
+  // reject patches that keep the build green but add new warnings.
+  // This gate exists because 96b46cd landed unused imports through the
+  // green-only gate — compiled fine, left a permanent dead-code warning.
+  log(`baseline cargo check (warning count)...`);
+  let baselineWarnings = (await cargoCheck()).warnings;
+  log(`baseline warnings = ${baselineWarnings}`);
+
+  let committedFiles = 0;
+  let revertedFiles = 0;
+  let warningReverts = 0;
+  let rationaleReverts = 0;
+
+  for (const r of eligible.slice(0, MAX_FILES)) {
+    log(`${r.file} (conf_avg=${r.confidence_avg} tier=${r.gradient_tier})`);
+    const full = `${REPO}/${r.file}`;
+    if (!existsSync(full)) { log(`  skip — file not found on disk`); continue; }
+
+    const source = await readFile(full, "utf8");
+    const patches = await requestPatches(r.file, source, r.suggestions_preview ?? "");
+
+    if (patches.length === 0) {
+      log(`  no patches produced`);
+      await auditLog({ action: "no_patches", file: r.file, reviewer_model: r.accepted_model });
+      continue;
+    }
+
+    log(`  ${patches.length} candidate patches`);
+    const { applied, rejected } = await applyPatches(r.file, patches);
+    log(`  applied ${applied}, rejected ${rejected.length}`);
+    for (const rj of rejected) log(`    ✗ ${rj.reason}`);
+
+    if (applied === 0) {
+      await auditLog({ action: "all_rejected", file: r.file, rejected: rejected.map(x => x.reason) });
+      continue;
+    }
+
+    log(`  running cargo check...`);
+    const { green, warnings } = await cargoCheck();
+    if (!green) {
+      log(`  ✗ build red — reverting ${r.file}`);
+      await revertFile(r.file);
+      revertedFiles++;
+      await auditLog({ action: "build_red_reverted", file: r.file, patches_applied: applied });
+      continue;
+    }
+
+    // New-warning gate. If a patch keeps the build green but adds a
+    // warning (unused import, dead_code after removed allow, etc), treat
+    // it the same as build red: revert. The cargo-green gate alone let
+    // 96b46cd land unused imports; this closes that door.
+    if (warnings > baselineWarnings) {
+      log(`  ✗ warnings ${baselineWarnings} → ${warnings} (+${warnings - baselineWarnings}) — reverting ${r.file}`);
+      await revertFile(r.file);
+      warningReverts++;
+      await auditLog({
+        action: "warnings_increased_reverted",
+        file: r.file,
+        patches_applied: applied,
+        warnings_before: baselineWarnings,
+        warnings_after: warnings,
+      });
+      continue;
+    }
+
+    // Rationale-diff alignment heuristic. At least one non-stopword
+    // token from the applied patches' combined new_string must appear
+    // in the rationale, OR the rationale must name the file's module.
+    // This catches cases like "Add destructive SQL filter" rationale
+    // with a diff that only adds `use tracing`.
+    const applied_patches = patches.slice(0, applied);
+    const rationale_ok = applied_patches.some((p) => rationaleMatchesDiff(p));
+    if (!rationale_ok) {
+      log(`  ✗ rationale doesn't name anything in diff — reverting ${r.file}`);
+      await revertFile(r.file);
+      rationaleReverts++;
+      await auditLog({
+        action: "rationale_mismatch_reverted",
+        file: r.file,
+        patches_applied: applied,
+        rationales: applied_patches.map((p) => p.rationale),
+      });
+      continue;
+    }
+
+    log(`  ✓ build green (warnings ${warnings}/${baselineWarnings})`);
+
+    // Dry-run MUST revert the file after a successful check so the
+    // workspace doesn't accumulate unpushed changes across dry-runs.
+    // Previously this left modified files, silently polluting state.
+    if (!COMMIT) {
+      log(`  (dry-run) would commit ${r.file} — reverting workspace change`);
+      await revertFile(r.file);
+      await auditLog({
+        action: "dry_run_would_commit",
+        file: r.file,
+        patches_applied: applied,
+        patches_rejected: rejected.length,
+        confidence_avg: r.confidence_avg,
+        gradient_tier: r.gradient_tier,
+        reviewer_model: r.accepted_model,
+      });
+      continue;
+    }
+
+    const ok = await gitCommit(r.file, applied_patches);
+    if (ok) {
+      committedFiles++;
+      // Advance the warning baseline so a later patch on a different
+      // file is compared against the new (committed) state, not the
+      // pre-run state — otherwise every subsequent patch trips the gate.
+      baselineWarnings = warnings;
+      await auditLog({
+        action: "committed",
+        file: r.file,
+        patches_applied: applied,
+        patches_rejected: rejected.length,
+        confidence_avg: r.confidence_avg,
+        gradient_tier: r.gradient_tier,
+        reviewer_model: r.accepted_model,
+        warnings_after: warnings,
+      });
+    }
+  }
+
+  log(`DONE · committed=${committedFiles} build_red=${revertedFiles} warning_revert=${warningReverts} rationale_revert=${rationaleReverts}`);
+}
+
+await main();

truth/README.md

@@ -0,0 +1,71 @@
+# Truth rules — file-backed policy
+
+Phase 42 PRD: *"truth/ dir at repo root — rule files, versioned in git."*
+
+This directory is the canonical home for TruthStore rules loaded from
+disk. Each `*.toml` file holds a set of `TruthRule` records for one
+task class. The truth crate's `load_from_dir(path)` walks this
+directory, parses every `.toml` file, and registers the rules it finds.
+
+## Structure
+
+```
+truth/
+├── README.md                 ← this file
+├── staffing.fill.toml        ← rules for task_class="staffing.fill"
+└── staffing.any.toml         ← rules for task_class="staffing.any"
+```
+
+File naming is informational — `load_from_dir` respects whatever
+`task_class` the rule declares internally, NOT the filename. Using
+task-class-matching filenames is a convention for humans reading the
+git tree.
+
+## Rule shape
+
+```toml
+[[rule]]
+id = "worker-active"
+task_class = "staffing.fill"
+description = "Worker must be active"
+condition = { type = "FieldEquals", field = "worker.status", value = "active" }
+action = { type = "Pass" }
+```
+
+`condition.type` is one of:
+  - `Always` — always true
+  - `FieldEquals { field, value }`
+  - `FieldMismatch { field, value }`
+  - `FieldEmpty { field }`
+  - `FieldGreater { field, threshold }`
+  - `FieldContainsAny { field, needles }`
+
+`action.type` is one of:
+  - `Pass` — rule informational; no enforcement
+  - `Reject { message }` — short-circuit with error
+  - `Redact { fields }` — mutate the context, strip fields
+  - `Block { message }` — hard stop, alert
+
+## Composition
+
+The crate's `default_truth_store()` continues to register rules
+**in code** for backward-compat. Operators can layer file-backed
+rules ON TOP via `load_from_dir`:
+
+```rust
+let store = truth::default_truth_store();
+let store = truth::load_from_dir(&store, "/home/profit/lakehouse/truth")?;
+```
+
+File-loaded rules are additive — they do NOT replace in-code rules.
+This lets the staffing team tune rules at the file level (edit a
+threshold, add a new `FieldContainsAny` blocklist) without waiting
+for a code deploy.
+
+## Explicit non-goals
+
+- **No hot reload** — per Phase 42 PRD ("Truth reload is explicit
+  in this phase"). Operators bounce the gateway or POST `/v1/context`
+  refresh endpoint (future) to pick up changes.
+- **No inheritance** — each file stands alone; rule IDs must be unique
+  across all files. Duplicate-ID detection is a load-time error.

truth/staffing.any.toml

@@ -0,0 +1,20 @@
+# Phase 42 — staffing.any task class rules (file-backed).
+#
+# Rules that apply across ALL staffing task classes (fill, rescue,
+# sms_draft, etc). The router registers them once and evaluates them
+# on every staffing.* call.
+
+[[rule]]
+id = "any.no-destructive-sql-in-context"
+task_class = "staffing.any"
+description = "Reject staffing calls whose SQL context contains destructive verbs"
+action = { type = "Reject", message = "destructive SQL rejected by staffing.any gate" }
+
+[rule.condition]
+type = "FieldContainsAny"
+field = "sql_context"
+needles = ["drop table", "truncate", "delete from", "drop schema", "drop database"]
+
+# Additional staffing.any rules (e.g. PII scrubbing, rate limits) can be
+# layered here; see the in-code staffing_rules() for the pii-redact rule
+# that ships by default.

truth/staffing.fill.toml

@@ -0,0 +1,32 @@
+# Phase 42 — staffing.fill task class rules (file-backed).
+#
+# These rules mirror what crates/truth/src/staffing.rs registers in code
+# via staffing_rules(). Both sets load at gateway startup; rule IDs MUST
+# be unique across the combined set, so operator-file overrides require
+# changing the in-code registration (or vice versa).
+#
+# Edit a threshold / add a needle / change a description? File-level
+# edits land without a code deploy. Schema changes (new RuleCondition
+# variants) still need a code bump.
+
+[[rule]]
+id = "fill.endorsed-count-matches-target"
+task_class = "staffing.fill"
+description = "endorsed_names.length must equal target_count"
+action = { type = "Reject", message = "endorsed count does not match target_count" }
+
+[rule.condition]
+type = "FieldEmpty"
+field = "contract.target_count"
+# When target_count is absent entirely, the rule fires and we reject.
+# Callers must include target_count; this is the first gate.
+
+[[rule]]
+id = "fill.city-required"
+task_class = "staffing.fill"
+description = "contract.target_city must be present and non-empty"
+action = { type = "Reject", message = "target_city is required" }
+
+[rule.condition]
+type = "FieldEmpty"
+field = "contract.target_city"

ui/server.ts

@@ -103,6 +103,93 @@ const REFACTOR_PHRASES = [
   "split this file", "too large",
 ];
 
+// Signal-class classifier — per file, given 2+ consecutive iterations'
+// reviews, tag the file's behavior:
+//   CONVERGING  — resolved > novel, score ↑
+//   LOOPING     — 3+ same findings repeat, novel = 0, score flat
+//   ORBITING    — novel findings each iter, no resolved (healthy depth)
+//   PLATEAU     — score flat + findings flat (diminishing returns)
+//   MIXED       — partial/unclear
+// This is the foundation for iter-6+ auto-routing: each class gets a
+// different sub-pipeline (specialist model, reviewer rotation, etc).
+const SIGNAL_PHRASES = [
+  "pseudocode", "placeholder", "stub", "unwired", "missing", "dead code", "orphaned",
+  "duplicate", "redundant", "refactor", "rewrite", "remove", "unused", "unnecessary",
+];
+
+async function signalClasses(): Promise<any> {
+  const runsDir = `${REPO}/tests/real-world/runs`;
+  // Load every review, group by file, sort by timestamp
+  const perFile: Record<string, Array<{run: string, phrases: Set<string>, score: number | null, conf_avg: number | null, findings: number, ts: number}>> = {};
+  try {
+    const dirs = await Array.fromAsync(new Bun.Glob("scrum_*").scan({ cwd: runsDir, onlyFiles: false }));
+    for (const d of dirs) {
+      const files = await Array.fromAsync(new Bun.Glob("review_*.json").scan({ cwd: `${runsDir}/${d}` }));
+      for (const f of files) {
+        try {
+          const p = `${runsDir}/${d}/${f}`;
+          const j = JSON.parse(await Bun.file(p).text());
+          const key = j.file?.replace("/home/profit/lakehouse/", "") ?? "?";
+          const sug = (j.suggestions ?? "").toLowerCase();
+          const phrases = new Set<string>();
+          for (const ph of SIGNAL_PHRASES) if (sug.includes(ph)) phrases.add(ph);
+          const scoreMatch = sug.match(/(\d(?:\.\d)?)\s*\/\s*10\b/);
+          const score = scoreMatch ? parseFloat(scoreMatch[1]) : null;
+          const mconf = [...sug.matchAll(/(?:confidence[*:\s]*\s*|\|\s*)(\d{1,3})\s*%/gi)].map(m=>parseInt(m[1],10));
+          const jconf = [...sug.matchAll(/"confidence"\s*:\s*(\d{1,3})(?!\d)/gi)].map(m=>parseInt(m[1],10));
+          const all = [...mconf, ...jconf].filter(x => 0 <= x && x <= 100);
+          const conf_avg = all.length ? Math.round(all.reduce((a,b)=>a+b,0)/all.length) : null;
+          const ts = (await Bun.file(p).stat()).mtime.getTime();
+          (perFile[key] ??= []).push({ run: d, phrases, score, conf_avg, findings: all.length, ts });
+        } catch {}
+      }
+    }
+  } catch (e) {
+    return { error: String(e), classes: {} };
+  }
+
+  const classes: Record<string, any> = {};
+  for (const [file, runs] of Object.entries(perFile)) {
+    runs.sort((a, b) => a.ts - b.ts);
+    if (runs.length < 2) { classes[file] = { cls: "NEW", runs: runs.length }; continue; }
+    const last = runs[runs.length - 1];
+    const prev = runs[runs.length - 2];
+    const novel = [...last.phrases].filter(p => !prev.phrases.has(p));
+    const resolved = [...prev.phrases].filter(p => !last.phrases.has(p));
+    const looping = [...prev.phrases].filter(p => last.phrases.has(p));
+    const dScore = (last.score != null && prev.score != null) ? last.score - prev.score : null;
+    const dConf = (last.conf_avg != null && prev.conf_avg != null) ? last.conf_avg - prev.conf_avg : null;
+    const dFindings = last.findings - prev.findings;
+
+    let cls: string;
+    if (dScore != null && dScore > 0 && resolved.length > novel.length) cls = "CONVERGING";
+    else if (looping.length >= 3 && novel.length === 0 && (dScore == null || Math.abs(dScore) < 0.5)) cls = "LOOPING";
+    else if (novel.length >= 2 && resolved.length === 0) cls = "ORBITING";
+    else if (Math.abs(dFindings) <= 1 && (dScore == null || Math.abs(dScore) < 0.5)) cls = "PLATEAU";
+    else cls = "MIXED";
+
+    classes[file] = {
+      cls,
+      runs: runs.length,
+      iter_span: `${runs[0].run}…${last.run}`,
+      prev_score: prev.score,
+      last_score: last.score,
+      delta_score: dScore,
+      delta_conf: dConf,
+      delta_findings: dFindings,
+      novel,
+      resolved,
+      looping,
+    };
+  }
+
+  // Summary counts
+  const counts: Record<string, number> = {};
+  for (const v of Object.values(classes)) counts[v.cls] = (counts[v.cls] ?? 0) + 1;
+
+  return { generated_at: new Date().toISOString(), counts, classes };
+}
+
 async function refactorSignals(): Promise<any> {
   // Walk every accepted review across all scrum runs. For each file,
   // count how many times its suggestions mention a refactor phrase.
@@ -262,6 +349,44 @@ Bun.serve({
     if (path === "/data/outcomes") return Response.json(await tailJsonl(`${KB}/outcomes.jsonl`, 30));
     if (path === "/data/audit_facts") return Response.json(await tailJsonl(`${KB}/audit_facts.jsonl`, 30));
 
+    // Pathway memory — consensus-designed sidecar (2026-04-24). Two
+    // exposed metrics: reuse_rate (activity — is it firing?) and
+    // avg_rungs_saved_per_commit (value — is it earning its keep?).
+    // Round-3 consensus (qwen3.5:397b) pointed out that activity
+    // without value tells us nothing; the UI needs both to judge the
+    // health of the hot-swap learning loop.
+    if (path === "/data/pathway_stats") {
+      try {
+        const r = await fetch("http://localhost:3100/vectors/pathway/stats", { signal: AbortSignal.timeout(3000) });
+        if (!r.ok) return Response.json({ error: `vectord ${r.status}`, stats: null });
+        const stats = await r.json();
+        // Tail recent scrum events to compute avg_rungs_saved_per_commit
+        // (a committed review = any row in scrum_reviews.jsonl; rungs_saved
+        // only populates when pathway memory fired AND the recommended
+        // model actually produced the accept).
+        const reviews = await tailJsonl(`${KB}/scrum_reviews.jsonl`, 200);
+        let totalCommits = 0;
+        let totalRungsSaved = 0;
+        let hotSwapHits = 0;
+        for (const r of reviews) {
+          totalCommits++;
+          if (r.pathway_hot_swap_hit) hotSwapHits++;
+          if (typeof r.rungs_saved === "number") totalRungsSaved += r.rungs_saved;
+        }
+        return Response.json({
+          stats,
+          scrum_window: {
+            reviews: totalCommits,
+            hot_swap_hits: hotSwapHits,
+            pathway_reuse_rate: totalCommits ? hotSwapHits / totalCommits : 0,
+            avg_rungs_saved_per_commit: totalCommits ? totalRungsSaved / totalCommits : 0,
+          },
+        });
+      } catch (e) {
+        return Response.json({ error: (e as Error).message, stats: null });
+      }
+    }
+
     if (path.startsWith("/data/file/")) {
       const relpath = decodeURIComponent(path.slice("/data/file/".length));
       return Response.json(await fileHistory(relpath));
@@ -269,6 +394,9 @@ Bun.serve({
     if (path === "/data/refactor_signals") {
       return Response.json(await refactorSignals());
     }
+    if (path === "/data/signal_classes") {
+      return Response.json(await signalClasses());
+    }
     if (path === "/data/search") {
       const q = url.searchParams.get("q") ?? "";
       return Response.json(await reverseIndex(q, 30));

ui/ui.css

@@ -405,3 +405,36 @@ main {
 .traj-hit-meta { font-family: var(--mono); font-size: 10px; color: var(--fg-muted); }
 .traj-hit-snip { font-family: var(--mono); font-size: 11px; color: var(--fg-dim); line-height: 1.5; }
 
+/* signal classes */
+.signal-class-row { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; }
+.signal-chip {
+  font-family: var(--mono); font-size: 10px; letter-spacing: 0.08em;
+  padding: 4px 10px; border: 1px solid var(--border); font-weight: 700;
+}
+.signal-converging { color: var(--green);  border-color: var(--green); }
+.signal-looping    { color: var(--red);    border-color: var(--red); }
+.signal-orbiting   { color: var(--purple); border-color: var(--purple); }
+.signal-plateau    { color: var(--yellow); border-color: var(--yellow); }
+.signal-mixed      { color: var(--blue);   border-color: var(--blue); }
+.signal-new        { color: var(--fg-muted); border-color: var(--fg-muted); }
+.signal-grid {
+  display: grid; grid-template-columns: repeat(auto-fill, minmax(480px, 1fr)); gap: 8px;
+}
+.signal-card {
+  border: 1px solid var(--border); background: var(--bg-1);
+  padding: 10px 12px; cursor: pointer; border-left: 3px solid var(--fg-muted);
+}
+.signal-card.signal-converging { border-left-color: var(--green); }
+.signal-card.signal-looping    { border-left-color: var(--red); }
+.signal-card.signal-orbiting   { border-left-color: var(--purple); }
+.signal-card.signal-plateau    { border-left-color: var(--yellow); }
+.signal-card.signal-mixed      { border-left-color: var(--blue); }
+.signal-card:hover { border-color: var(--border-hi); background: var(--bg-2); }
+.signal-card-top { display: flex; gap: 10px; align-items: center; margin-bottom: 6px; }
+.signal-card-file { font-family: var(--mono); font-size: 11px; color: var(--fg); }
+.signal-card-body { font-family: var(--mono); font-size: 10px; color: var(--fg-dim); line-height: 1.55; }
+.signal-card-body > div { margin-top: 2px; }
+.signal-novel    { color: var(--purple); }
+.signal-resolved { color: var(--green); }
+.signal-loop     { color: var(--red); }
+

ui/ui.js

@@ -444,6 +444,54 @@ async function drawTrajectory() {
   const statsEl = document.getElementById("traj-stats");
   clear(statsEl);
 
+  // SECTION 0 — signal classes (CONVERGING/LOOPING/ORBITING/PLATEAU/MIXED)
+  try {
+    const sc = await fetch("/data/signal_classes").then(r => r.json());
+    body.append(el("div", { className: "traj-section-head", text: "SIGNAL CLASSES · iter-to-iter behavior per file" }));
+    body.append(el("div", { className: "traj-section-explain", text:
+      "Each file compared iter-to-iter: CONVERGING = fix landed (resolved > novel + score↑), " +
+      "LOOPING = same findings repeating (deadlock candidate for hyper-focus), " +
+      "ORBITING = novel findings every iter (healthy depth-first), " +
+      "PLATEAU = score+findings flat (diminishing returns, needs different angle), " +
+      "MIXED = partial movement, NEW = only 1 iter so far."
+    }));
+    const classRow = el("div", { className: "signal-class-row" });
+    for (const [cls, n] of Object.entries(sc.counts ?? {})) {
+      const chip = el("span", { className: `signal-chip signal-${cls.toLowerCase()}`, text: `${cls} ${n}` });
+      classRow.append(chip);
+    }
+    body.append(classRow);
+    const grid = el("div", { className: "signal-grid" });
+    const sorted = Object.entries(sc.classes ?? {}).sort((a, b) => {
+      const order = { CONVERGING: 0, LOOPING: 1, ORBITING: 2, MIXED: 3, PLATEAU: 4, NEW: 5 };
+      return (order[a[1].cls] ?? 9) - (order[b[1].cls] ?? 9);
+    });
+    for (const [file, info] of sorted) {
+      const card = el("div", { className: `signal-card signal-${info.cls.toLowerCase()}` });
+      card.append(el("div", { className: "signal-card-top" },
+        el("span", { className: `signal-chip signal-${info.cls.toLowerCase()}`, text: info.cls }),
+        el("span", { className: "signal-card-file", text: file })
+      ));
+      const body2 = el("div", { className: "signal-card-body" });
+      if (info.prev_score != null || info.last_score != null) {
+        body2.append(el("div", { text: `score ${info.prev_score ?? "?"} → ${info.last_score ?? "?"} (Δ ${info.delta_score != null ? (info.delta_score > 0 ? "+" : "") + info.delta_score.toFixed(1) : "?"})` }));
+      }
+      if (info.novel?.length) body2.append(el("div", { className: "signal-novel", text: `NEW: ${info.novel.join(", ")}` }));
+      if (info.resolved?.length) body2.append(el("div", { className: "signal-resolved", text: `RESOLVED: ${info.resolved.join(", ")}` }));
+      if (info.looping?.length) body2.append(el("div", { className: "signal-loop", text: `LOOPING: ${info.looping.join(", ")}` }));
+      card.append(body2);
+      card.addEventListener("click", () => {
+        state.selected = { type: "file", id: `/home/profit/lakehouse/${file}` };
+        renderContext();
+        document.querySelector('#views button[data-view="trace"]').click();
+      });
+      grid.append(card);
+    }
+    body.append(grid);
+  } catch (e) {
+    body.append(el("div", { className: "ctx-hint", text: `signal classes error: ${e}` }));
+  }
+
   // SECTION 1 — refactor signals
   const sig = await fetch("/data/refactor_signals").then(r => r.json());
   const sigs = sig.signals ?? [];
@@ -541,6 +589,37 @@ function metricBox(label, big, kind, opts = {}) {
 function drawMetrics() {
   const grid = document.getElementById("metric-grid");
   clear(grid);
+  // Kick off pathway fetch in parallel; render when it resolves so the
+  // rest of the metrics grid appears immediately. The cards append to
+  // the grid after the synchronous block below — they'll show up at
+  // the bottom of the grid within a tick of first render.
+  fetch("/data/pathway_stats").then(r => r.ok ? r.json() : null).then(j => {
+    if (!j || !j.stats) return;
+    const s = j.stats;
+    const w = j.scrum_window ?? {};
+    // Activity metric — is the hot-swap firing at all?
+    grid.append(metricBox("pathway reuse rate", `${Math.round((w.pathway_reuse_rate ?? 0) * 100)}%`,
+      (w.pathway_reuse_rate ?? 0) > 0.1 ? "good" : (w.pathway_reuse_rate ?? 0) > 0 ? "warn" : "bad", {
+      explain: "% of recent reviews where a pathway hot-swap fired (narrow fingerprint match + 0.80 success rate + ≥3 replays + audit_consensus pass + 0.90 similarity).",
+      source: `scrum_reviews.jsonl .pathway_hot_swap_hit over last ${w.reviews ?? 0} reviews (${w.hot_swap_hits ?? 0} hits)`,
+      good: "≥10% sustained = index earning its keep. <10% over many iters = fingerprint too narrow or probation too strict. 0% on fresh install is expected (no replays yet).",
+    }));
+    // Value metric — how much compute did hot-swap actually save?
+    const saved = w.avg_rungs_saved_per_commit ?? 0;
+    grid.append(metricBox("avg rungs saved", saved.toFixed(2),
+      saved >= 1 ? "good" : saved > 0 ? "warn" : "bad", {
+      explain: "Average ladder rungs skipped per committed review by hot-swap. Rungs_saved = recommended_rung - 1 when the recommended model succeeded (otherwise 0).",
+      source: "scrum_reviews.jsonl .rungs_saved averaged",
+      good: "Every 1.0 here ≈ one less model call per review. At 21 files/iter, 1.0 saved = 21 cloud calls avoided. Value only counts when the replay actually succeeded.",
+    }));
+    // Stability metric — retired pathways indicate the learning loop is correcting itself.
+    grid.append(metricBox("pathways tracked", String(s.total_pathways),
+      s.total_pathways > 0 ? "good" : "warn", {
+      explain: `Total pathway traces stored. ${s.retired} retired (below 0.80 success after ≥3 replays). ${s.with_audit_pass} audit-passed, eligible for hot-swap probation.`,
+      source: "/vectors/pathway/stats",
+      good: `Grows monotonically with scrum runs. Retired=${s.retired} is HEALTHY — it means the learning loop is pruning pathways that stopped working. replay_success_rate=${(s.replay_success_rate*100).toFixed(0)}% aggregates all historical replays.`,
+    }));
+  }).catch(() => {});
   const byTier = { auto:0, dry_run:0, simulation:0, block:0, unknown:0 };
   state.reviews.forEach(r => { const t = r.gradient_tier ?? "unknown"; if (byTier[t] != null) byTier[t]++; });
   const total = state.reviews.length || 1;