profit/lakehouse
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Phase 19 wiring + Path 1/2 work + chain integrity fixes

Browse Source 
Backend:
- crates/vectord/src/playbook_memory.rs (new): Phase 19 in-memory boost
  store with seed/rebuild/snapshot, plus temporal decay (e^-age/30 per
  playbook), persist_to_sql endpoint backing successful_playbooks_live,
  and discover_patterns endpoint for meta-index pattern aggregation
  (recurring certs/skills/archetype/reliability across similar past fills).
- DEFAULT_TOP_K_PLAYBOOKS bumped 5 → 25; old default silently missed
  most boosts when memory had > 25 entries.
- service.rs: new routes /vectors/playbook_memory/{seed,rebuild,stats,
  persist_sql,patterns}.

Bun staffing co-pilot (mcp-server/):
- /search, /match, /verify, /proof, /simulation/run, MCP tools all
  forward use_playbook_memory:true and playbook_memory_k:25 to the
  hybrid endpoint. Boost was previously dark across the entire app.
- /log no longer POSTs to /ingest/file — that endpoint REPLACES the
  dataset's object list, so single-row CSV writes were wiping all prior
  rows in successful_playbooks (sp_rows went 33→1 in one /log call).
  /log now seeds playbook_memory with canonical short text and calls
  /persist_sql to keep successful_playbooks_live in sync.
- /simulation/run cumulative end-of-week CSV write removed for the same
  reason. Per-day per-contract /seed (added in this session) is the
  accumulating feedback path now.
- search.html addWorkerInsight renders a green "Endorsed · N playbooks"
  chip with playbook citations when boost > 0.

Internal Dioxus UI (crates/ui/):
- Dashboard phase list rewritten through Phase 19 (was stuck at "Phase
  16: File Watcher" / "Phase 17: DB Connector" — both wrong).
- Removed fabricated "27ms" stat label.
- Ask tab examples + SQL default replaced with real staffing prompts
  against candidates/clients/job_orders (was referencing nonexistent
  employees/products/events).
- New Playbook tab exposes /vectors/playbook_memory/{stats,rebuild} and
  side-by-side hybrid search (boost OFF vs ON) with citations.

Tests (tests/multi-agent/):
- run_e2e_rated.ts: parallel two-agent (mistral + qwen2.5) build phase
  + verifier rating (geo, auth, persist, boost, speed → /10).
- network_proving.ts: continuous build → verify → repeat with
  staffing-recruiter profile hot-swap; geo-discrimination check.
- chain_of_custody.ts: single recruiter operation traced through every
  layer (Bun /search, direct /vectors/hybrid parity, /log, SQL,
  playbook_memory growth, profile activation, post-op boost lift).
This commit is contained in:
root
2026-04-20 06:21:13 -05:00
parent 8e3cac5812
commit 25b7e6c3a7
13 changed files with 4566 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ui/src/main.rs
+429 -32
View File
@@ -178,9 +178,116 @@ enum Tab {
Explore,
Sql,
Ingest,
Playbook,
Status,
}
// --- Playbook memory types (Phase 19) ---
#[derive(Debug, Clone, Deserialize, PartialEq)]
struct PlaybookStats {
entries: usize,
entries_with_embeddings: usize,
#[serde(default)]
total_names_endorsed: usize,
#[serde(default)]
sample: Vec<PlaybookSample>,
}
#[derive(Debug, Clone, Deserialize, PartialEq)]
struct PlaybookSample {
id: String,
operation: String,
#[serde(default)]
city: Option<String>,
#[serde(default)]
state: Option<String>,
#[serde(default)]
endorsed: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, PartialEq)]
struct HybridResp {
#[serde(default)]
sql_matches: usize,
#[serde(default)]
vector_reranked: usize,
#[serde(default)]
method: String,
#[serde(default)]
duration_ms: u64,
#[serde(default)]
answer: Option<String>,
#[serde(default)]
sources: Vec<HybridSource>,
}
#[derive(Debug, Clone, Deserialize, PartialEq)]
struct HybridSource {
doc_id: String,
chunk_text: String,
score: f32,
#[serde(default)]
sql_verified: bool,
#[serde(default)]
playbook_boost: f32,
#[serde(default)]
playbook_citations: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, PartialEq)]
struct IndexInfo {
index_name: String,
source: String,
#[serde(default)]
chunk_count: usize,
#[serde(default)]
vector_backend: String,
}
async fn fetch_playbook_stats() -> Result<PlaybookStats, String> {
let resp = reqwest::get(&format!("{}/vectors/playbook_memory/stats", api_base()))
.await.map_err(|e| e.to_string())?;
if !resp.status().is_success() {
return Err(format!("HTTP {}: {}", resp.status(), resp.text().await.unwrap_or_default()));
}
resp.json().await.map_err(|e| e.to_string())
}
async fn rebuild_playbook_memory() -> Result<serde_json::Value, String> {
let client = reqwest::Client::new();
let resp = client.post(&format!("{}/vectors/playbook_memory/rebuild", api_base()))
.json(&serde_json::json!({}))
.send().await.map_err(|e| e.to_string())?;
if !resp.status().is_success() {
return Err(format!("HTTP {}: {}", resp.status(), resp.text().await.unwrap_or_default()));
}
resp.json().await.map_err(|e| e.to_string())
}
async fn fetch_indexes() -> Result<Vec<IndexInfo>, String> {
let resp = reqwest::get(&format!("{}/vectors/indexes", api_base()))
.await.map_err(|e| e.to_string())?;
resp.json().await.map_err(|e| e.to_string())
}
async fn hybrid_search(index_name: &str, question: &str, use_playbook: bool, top_k: usize) -> Result<HybridResp, String> {
let client = reqwest::Client::new();
let resp = client.post(&format!("{}/vectors/hybrid", api_base()))
.json(&serde_json::json!({
"index_name": index_name,
"question": question,
"top_k": top_k,
"generate": false,
"use_playbook_memory": use_playbook,
}))
.send().await.map_err(|e| e.to_string())?;
if !resp.status().is_success() {
return Err(format!("HTTP {}: {}", resp.status(), resp.text().await.unwrap_or_default()));
}
resp.json().await.map_err(|e| e.to_string())
}
// --- App ---
#[component]
@@ -239,6 +346,11 @@ fn App() -> Element {
onclick: move |_| active_tab.set(Tab::Ingest),
"Ingest"
}
button {
class: if *active_tab.read() == Tab::Playbook { "tab active" } else { "tab" },
onclick: move |_| active_tab.set(Tab::Playbook),
"Playbook"
}
button {
class: if *active_tab.read() == Tab::Status { "tab active" } else { "tab" },
onclick: move |_| active_tab.set(Tab::Status),
@@ -260,6 +372,7 @@ fn App() -> Element {
Tab::Explore => rsx! { ExplorePanel { datasets: datasets.read().clone() } },
Tab::Sql => rsx! { SqlPanel {} },
Tab::Ingest => rsx! { IngestPanel {} },
Tab::Playbook => rsx! { PlaybookPanel {} },
Tab::Status => rsx! { StatusPanel {} },
}
}
@@ -354,14 +467,14 @@ fn AskPanel(datasets: Vec<Dataset>) -> Element {
div { class: "panel ask-panel",
div { class: "ask-hero",
h2 { "Ask your data anything" }
p { class: "subtitle", "Natural language → SQL → Results. Powered by local AI." }
p { class: "subtitle", "Natural language → SQL → Results. Powered by local AI over the staffing dataset." }
}
div { class: "ask-input-row",
input {
class: "ask-input",
value: "{question}",
placeholder: "e.g. Which department has the highest average salary?",
placeholder: "e.g. Which clients placed the most candidates last quarter?",
oninput: move |e| question.set(e.value()),
onkeydown: move |e| {
if e.key() == Key::Enter {
@@ -432,10 +545,12 @@ fn AskPanel(datasets: Vec<Dataset>) -> Element {
div { class: "ask-examples",
"Try: "
button { class: "example-btn", onclick: move |_| question.set("Which department has the highest average salary?".into()), "highest avg salary by dept" }
button { class: "example-btn", onclick: move |_| question.set("Show me the top 3 most expensive products".into()), "top 3 expensive products" }
button { class: "example-btn", onclick: move |_| question.set("How many events per action type?".into()), "events by action" }
button { class: "example-btn", onclick: move |_| question.set("List all employees who earn more than 90000".into()), "employees > 90k" }
button { class: "example-btn", onclick: move |_| question.set("How many candidates do we have by city?".into()), "candidates by city" }
button { class: "example-btn", onclick: move |_| question.set("Top 10 clients by total placements".into()), "top clients by placements" }
button { class: "example-btn", onclick: move |_| question.set("Open job orders ordered by bill rate descending".into()), "open jobs by rate" }
button { class: "example-btn", onclick: move |_| question.set("Recruiters with the highest placement count".into()), "top recruiters" }
button { class: "example-btn", onclick: move |_| question.set("Total billed hours per client last month".into()), "hours per client" }
button { class: "example-btn", onclick: move |_| question.set("Cold leads: candidates we called more than 5 times but never placed".into()), "cold leads" }
}
if let Some(sql) = generated_sql.read().as_ref() {
@@ -578,7 +693,7 @@ fn ExplorePanel(datasets: Vec<Dataset>) -> Element {
#[component]
fn SqlPanel() -> Element {
let mut query_text = use_signal(|| String::from("SELECT * FROM employees LIMIT 10"));
let mut query_text = use_signal(|| String::from("SELECT candidate_id, first_name, last_name, city, status FROM candidates LIMIT 10"));
let mut result = use_signal(|| None::<Result<QueryResponse, String>>);
let mut loading = use_signal(|| false);
@@ -727,7 +842,7 @@ fn DashboardPanel() -> Element {
}
div { class: "stat-card accent",
div { class: "stat-value", "{s[\"hnsw_loaded\"]}" }
div { class: "stat-label", "HNSW Indexes (27ms)" }
div { class: "stat-label", "HNSW Indexes Loaded" }
}
div { class: "stat-card",
div { class: "stat-value", "{s[\"tools\"]}" }
@@ -750,27 +865,27 @@ fn DashboardPanel() -> Element {
div { class: "arch-grid",
div { class: "arch-card",
div { class: "arch-title", "Ingest" }
div { class: "arch-items", "CSV, JSON, PDF, Text, PostgreSQL, File Watcher" }
div { class: "arch-items", "CSV · JSON · PDF (+OCR) · Text · Postgres · MySQL · Inbox watcher · Cron schedules" }
}
div { class: "arch-card",
div { class: "arch-title", "Storage" }
div { class: "arch-items", "Parquet on Object Storage, Delta Writes, Compaction" }
div { class: "arch-items", "Parquet on Object Storage · Delta writes · Compaction · Tombstones · Multi-bucket federation + rescue" }
}
div { class: "arch-card",
div { class: "arch-title", "Query" }
div { class: "arch-items", "DataFusion SQL, MemCache (9.8x), Hot/Cold" }
div { class: "arch-items", "DataFusion SQL · MemCache (9.8× hot) · Merge-on-read · AI-safe views" }
}
div { class: "arch-card",
div { class: "arch-title", "AI" }
div { class: "arch-items", "Ollama (local), Embed, Generate, RAG, HNSW" }
div { class: "arch-title", "AI / Vector" }
div { class: "arch-items", "Ollama (local) · Embed/Generate/RAG · HNSW (Parquet) · Lance IVF_PQ · Hybrid SQL+vector · Profile-scoped" }
}
div { class: "arch-card",
div { class: "arch-title", "Learning loop" }
div { class: "arch-items", "Playbook memory · Endorsement boost · Multi-agent orchestrator · Autotune agent (Pareto-promote)" }
}
div { class: "arch-card",
div { class: "arch-title", "Governance" }
div { class: "arch-items", "Event Journal, PII Detection, Tool Registry, Access Control" }
}
div { class: "arch-card",
div { class: "arch-title", "Agents" }
div { class: "arch-items", "Workspaces, Handoff, Shortlists, Activity Logs" }
div { class: "arch-items", "Event journal · PII detection · Tool registry · Access control · Audit log · Catalog v2 metadata" }
}
}
}
@@ -779,20 +894,23 @@ fn DashboardPanel() -> Element {
h3 { "Build Progression" }
div { class: "phase-list",
{rsx! {
PhaseItem { num: "0-5", name: "Foundation", detail: "Storage, Catalog, DataFusion, AI, UI, gRPC" }
PhaseItem { num: "6", name: "Ingest Pipeline", detail: "CSV/JSON/PDF/Text auto-schema" }
PhaseItem { num: "7", name: "Vector + RAG", detail: "Embed, Search, LLM Answers" }
PhaseItem { num: "8", name: "Hot Cache", detail: "9.8x speedup, Delta Writes" }
PhaseItem { num: "8.5", name: "Agent Workspaces", detail: "Per-contract, Instant Handoff" }
PhaseItem { num: "9", name: "Event Journal", detail: "Append-only Mutation History" }
PhaseItem { num: "10", name: "Rich Catalog", detail: "PII Detection, Lineage" }
PhaseItem { num: "11", name: "Embedding Versioning", detail: "Model-proof Vectors" }
PhaseItem { num: "12", name: "Tool Registry", detail: "6 Governed Actions + Audit" }
PhaseItem { num: "13", name: "Access Control", detail: "Role-based, Field-level" }
PhaseItem { num: "14", name: "Schema Evolution", detail: "Diff Detection, AI Migration" }
PhaseItem { num: "15", name: "HNSW Index", detail: "100K Search in 27ms" }
PhaseItem { num: "16", name: "File Watcher", detail: "Auto-ingest from Inbox" }
PhaseItem { num: "17", name: "DB Connector", detail: "PostgreSQL Import" }
PhaseItem { num: "0-5", name: "Foundation", detail: "Storage · Catalog · DataFusion · Ollama · UI · gRPC" }
PhaseItem { num: "6", name: "Ingest Pipeline", detail: "CSV · JSON · PDF · Text · auto-schema · dedupe" }
PhaseItem { num: "7", name: "Vector + RAG", detail: "Embed · brute-force cosine · LLM grounded answers" }
PhaseItem { num: "8", name: "Hot Cache + Deltas", detail: "MemTable LRU · 9.8× speedup · merge-on-read · compaction" }
PhaseItem { num: "8.5", name: "Agent Workspaces", detail: "Per-contract · daily/weekly/monthly tiers · zero-copy handoff" }
PhaseItem { num: "9", name: "Event Journal", detail: "Append-only mutation log · time-travel · audit" }
PhaseItem { num: "10", name: "Rich Catalog v2", detail: "PII auto-detection · lineage · freshness SLA · sensitivity" }
PhaseItem { num: "11", name: "Embedding Versioning", detail: "Per-index model+version · A/B · incremental re-embed" }
PhaseItem { num: "12", name: "Tool Registry", detail: "Governed actions · param validation · audit · MCP-ready" }
PhaseItem { num: "13", name: "Access Control", detail: "Roles · field-level sensitivity · column masking · query audit" }
PhaseItem { num: "14", name: "Schema Evolution", detail: "Diff detection · AI migration prompts · versioned schemas" }
PhaseItem { num: "15", name: "HNSW + Trials", detail: "100K vectors · p50 873µs · trial journal · eval harness" }
PhaseItem { num: "16", name: "Hot-swap + Autotune", detail: "Promotion registry · rollback · ε-greedy agent · Pareto winner" }
PhaseItem { num: "17", name: "Model Profiles + VRAM", detail: "ModelProfile manifests · scoped search · sequential model swap" }
PhaseItem { num: "18", name: "Lance hybrid backend", detail: "IVF_PQ build 14× faster · random fetch 112× · S3-native · per-profile routing" }
PhaseItem { num: "19", name: "Playbook memory", detail: "Feedback loop · endorsement boost (cap 0.25) · orchestrator write-through · citations" }
PhaseItem { num: "+", name: "Federation + Schedules", detail: "Multi-bucket · rescue fallback · error journal · MySQL · PDF OCR · cron ingest · catalog dedupe" }
}}
}
}
@@ -816,6 +934,285 @@ fn PhaseItem(num: String, name: String, detail: String) -> Element {
}
}
// === PLAYBOOK — Phase 19 meta-index feedback loop ===
#[component]
fn PlaybookPanel() -> Element {
let mut stats = use_signal(|| None::<Result<PlaybookStats, String>>);
let mut indexes = use_signal(Vec::<IndexInfo>::new);
let mut rebuild_status = use_signal(|| None::<Result<String, String>>);
let mut rebuilding = use_signal(|| false);
let mut loaded = use_signal(|| false);
// Comparison state
let mut selected_index = use_signal(|| String::new());
let mut question = use_signal(|| String::from("reliable assembler in Detroit"));
let mut top_k = use_signal(|| 10usize);
let mut compare_loading = use_signal(|| false);
let mut hits_off = use_signal(|| None::<Result<HybridResp, String>>);
let mut hits_on = use_signal(|| None::<Result<HybridResp, String>>);
let load_all = move || {
spawn(async move {
stats.set(Some(fetch_playbook_stats().await));
if let Ok(ix) = fetch_indexes().await {
if selected_index.read().is_empty() {
if let Some(default) = ix.iter().find(|i| i.source == "workers_500k").or_else(|| ix.first()) {
selected_index.set(default.index_name.clone());
}
}
indexes.set(ix);
}
});
};
use_effect(move || {
if !*loaded.read() {
loaded.set(true);
load_all();
}
});
let do_rebuild = move |_| {
spawn(async move {
rebuilding.set(true);
rebuild_status.set(None);
match rebuild_playbook_memory().await {
Ok(v) => rebuild_status.set(Some(Ok(format!("rebuild ok — {}", v)))),
Err(e) => rebuild_status.set(Some(Err(e))),
}
// Refresh stats afterward
stats.set(Some(fetch_playbook_stats().await));
rebuilding.set(false);
});
};
let do_compare = move |_| {
let idx = selected_index.read().clone();
let q = question.read().clone();
let k = *top_k.read();
if idx.is_empty() || q.trim().is_empty() { return; }
spawn(async move {
compare_loading.set(true);
hits_off.set(None);
hits_on.set(None);
// Run both sequentially so the embedding cache is shared
hits_off.set(Some(hybrid_search(&idx, &q, false, k).await));
hits_on.set(Some(hybrid_search(&idx, &q, true, k).await));
compare_loading.set(false);
});
};
rsx! {
div { class: "panel",
div { class: "ask-hero",
h2 { "Playbook Memory" }
p { class: "subtitle",
"Phase 19 feedback loop: past successful playbooks boost future search rankings. \
Endorsed workers from semantically similar past operations re-rank toward the top, \
with citations back to the playbook that endorsed them."
}
}
// Stats card
div { class: "panel-section",
match stats.read().as_ref() {
None => rsx! { div { class: "loading", "loading playbook stats..." } },
Some(Err(e)) => rsx! { div { class: "error", "stats: {e}" } },
Some(Ok(s)) => rsx! {
div { class: "stat-grid",
div { class: "stat-card",
div { class: "stat-value", "{s.entries}" }
div { class: "stat-label", "Playbooks in Memory" }
}
div { class: "stat-card",
div { class: "stat-value", "{s.entries_with_embeddings}" }
div { class: "stat-label", "Embedded" }
}
div { class: "stat-card accent",
div { class: "stat-value", "{s.total_names_endorsed}" }
div { class: "stat-label", "Endorsed Worker-Tags" }
}
}
}
}
div { class: "sql-actions",
button {
class: "btn",
disabled: *rebuilding.read(),
onclick: do_rebuild,
if *rebuilding.read() { "rebuilding from successful_playbooks..." } else { "Rebuild from successful_playbooks" }
}
}
if let Some(s) = rebuild_status.read().as_ref() {
match s {
Ok(msg) => rsx! { div { class: "result-box", "{msg}" } },
Err(e) => rsx! { div { class: "error", "{e}" } },
}
}
}
// Sample playbooks
if let Some(Ok(s)) = stats.read().as_ref() {
if !s.sample.is_empty() {
div { class: "panel-section",
h3 { "Sample playbooks" }
div { class: "table-wrap",
table {
thead { tr {
th { "ID" }
th { "Operation" }
th { "Location" }
th { "Endorsed" }
} }
tbody {
for pb in s.sample.iter() {
{
let loc = match (&pb.city, &pb.state) {
(Some(c), Some(st)) => format!("{c}, {st}"),
_ => "—".into(),
};
let endorsed = if pb.endorsed.is_empty() {
"—".to_string()
} else {
pb.endorsed.join(", ")
};
let pid = pb.id.clone();
let op = pb.operation.clone();
rsx! {
tr {
td { class: "mono-cell", title: "{pid}", "{pid}" }
td { "{op}" }
td { "{loc}" }
td { "{endorsed}" }
}
}
}
}
}
}
}
}
}
}
// Side-by-side comparison: boost OFF vs ON
div { class: "panel-section",
h3 { "See the boost — search compared" }
p { class: "hint",
"Run the same query against the same index twice — once with playbook boost OFF and once ON. \
Hits with non-zero playbook_boost and citations are workers that past similar playbooks endorsed."
}
div { class: "form-row",
label { "Index" }
select {
value: "{selected_index}",
onchange: move |e| selected_index.set(e.value()),
for ix in indexes.read().iter() {
option { value: "{ix.index_name}", "{ix.index_name} ({ix.source}, {ix.chunk_count} chunks, {ix.vector_backend})" }
}
}
}
div { class: "form-row",
label { "Question" }
input {
value: "{question}",
oninput: move |e| question.set(e.value()),
placeholder: "e.g. reliable assembler in Detroit"
}
}
div { class: "form-row",
label { "Top K" }
input {
r#type: "number",
value: "{top_k}",
oninput: move |e| {
if let Ok(n) = e.value().parse::<usize>() { top_k.set(n.clamp(1, 50)); }
}
}
}
button {
class: "btn btn-ask",
disabled: *compare_loading.read(),
onclick: do_compare,
if *compare_loading.read() { "running both queries..." } else { "Run comparison" }
}
div { class: "explore-grid",
div { class: "ds-detail",
h3 { "Boost OFF (vanilla)" }
match hits_off.read().as_ref() {
None => rsx! { div { class: "empty", "—" } },
Some(Err(e)) => rsx! { div { class: "error", "{e}" } },
Some(Ok(r)) => rsx! { HybridHitTable { resp: r.clone() } },
}
}
div { class: "ds-detail",
h3 { "Boost ON (Phase 19)" }
match hits_on.read().as_ref() {
None => rsx! { div { class: "empty", "—" } },
Some(Err(e)) => rsx! { div { class: "error", "{e}" } },
Some(Ok(r)) => rsx! { HybridHitTable { resp: r.clone() } },
}
}
}
}
}
}
}
#[component]
fn HybridHitTable(resp: HybridResp) -> Element {
rsx! {
div { class: "results-info",
"{resp.sources.len()} hits · {resp.duration_ms}ms · method={resp.method}"
}
if resp.sources.is_empty() {
div { class: "empty-sm", "no hits" }
} else {
div { class: "table-wrap",
table {
thead { tr {
th { "#" }
th { "Doc" }
th { "Score" }
th { "Boost" }
th { "Citations" }
th { "Snippet" }
} }
tbody {
for (i, h) in resp.sources.iter().enumerate() {
{
let snippet: String = h.chunk_text.chars().take(120).collect();
let cites = if h.playbook_citations.is_empty() {
"—".to_string()
} else {
h.playbook_citations.join(", ")
};
let row_class = if h.playbook_boost > 0.0 { "boosted-row" } else { "" };
let rank = i + 1;
let did = h.doc_id.clone();
let score = format!("{:.3}", h.score);
let boost = if h.playbook_boost > 0.0 { format!("+{:.3}", h.playbook_boost) } else { "—".into() };
rsx! {
tr { class: "{row_class}",
td { "{rank}" }
td { class: "mono-cell", "{did}" }
td { "{score}" }
td { "{boost}" }
td { class: "mono-cell", title: "{cites}", "{cites}" }
td { "{snippet}" }
}
}
}
}
}
}
}
}
}
}
// === INGEST — Data on-ramp ===
#[component]