Observer KB UI: now shows LLM Team extracted facts + entities

- Knowledge tab: stats (categories, verified facts, entities, decisions, insights, questions), auto-generated categories, key decisions, key insights - Facts & Entities tab (NEW): 279 extracted facts from LLM Team extract mode, 264 entities as clickable chips (click → search), verified entries from secondary model - Search tab: semantic HNSW search across all knowledge - Ask AI tab: RAG with cited sources - Ingest tab: upload with progress - Entity chips link to semantic search - Category drill-down shows related content via HNSW Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
LLM Team integration: extract mode processes 8 conversations
2026-03-28 00:25:39 -05:00 · 2026-03-28 00:22:29 -05:00 · 2026-03-27 23:35:15 -05:00 · 2026-03-27 23:26:05 -05:00 · 2026-03-27 23:17:16 -05:00 · 2026-03-27 23:12:13 -05:00
119 changed files with 6688 additions and 229 deletions
@@ -695,7 +695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
 dependencies = [
 "chrono",
- "phf",
+ "phf 0.12.1",
 ]
 [[package]]
@@ -2220,6 +2220,12 @@ dependencies = [
 "num-traits",
 ]
 [[package]]
 name = "fallible-iterator"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 [[package]]
 name = "fastrand"
 version = "2.3.0"
@@ -2373,8 +2379,10 @@ name = "gateway"
 version = "0.1.0"
 dependencies = [
 "aibridge",
 "arrow",
 "axum",
 "catalogd",
 "chrono",
 "ingestd",
 "journald",
 "object_store",
@@ -2425,7 +2433,7 @@ dependencies = [
 "cfg-if",
 "js-sys",
 "libc",
- "wasi",
+ "wasi 0.11.1+wasi-snapshot-preview1",
 "wasm-bindgen",
 ]
@@ -2594,12 +2602,27 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 [[package]]
 name = "hermit-abi"
 version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
 [[package]]
 name = "hex"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 [[package]]
 name = "hmac"
 version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
 dependencies = [
 "digest",
 ]
 [[package]]
 name = "http"
 version = "1.4.0"
@@ -2906,6 +2929,7 @@ dependencies = [
 "shared",
 "storaged",
 "tokio",
 "tokio-postgres",
 "tracing",
 ]
@@ -2919,6 +2943,19 @@ dependencies = [
 "generic-array",
 ]
 [[package]]
 name = "instant-distance"
 version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c619cdaa30bb84088963968bee12a45ea5fbbf355f2c021bcd15589f5ca494a"
 dependencies = [
 "num_cpus",
 "ordered-float 3.9.2",
 "parking_lot",
 "rand 0.8.5",
 "rayon",
 ]
 [[package]]
 name = "integer-encoding"
 version = "3.0.4"
@@ -3153,6 +3190,15 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 [[package]]
 name = "libredox"
 version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08"
 dependencies = [
 "libc",
 ]
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -3381,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
 dependencies = [
 "libc",
- "wasi",
+ "wasi 0.11.1+wasi-snapshot-preview1",
 "windows-sys 0.61.2",
 ]
@@ -3548,6 +3594,16 @@ dependencies = [
 "libm",
 ]
 [[package]]
 name = "num_cpus"
 version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
 dependencies = [
 "hermit-abi",
 "libc",
 ]
 [[package]]
 name = "num_enum"
 version = "0.7.6"
@@ -3570,6 +3626,24 @@ dependencies = [
 "syn",
 ]
 [[package]]
 name = "objc2-core-foundation"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
 dependencies = [
 "bitflags",
 ]
 [[package]]
 name = "objc2-system-configuration"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396"
 dependencies = [
 "objc2-core-foundation",
 ]
 [[package]]
 name = "object"
 version = "0.37.3"
@@ -3686,6 +3760,15 @@ dependencies = [
 "num-traits",
 ]
 [[package]]
 name = "ordered-float"
 version = "3.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc"
 dependencies = [
 "num-traits",
 ]
 [[package]]
 name = "parking_lot"
 version = "0.12.5"
@@ -3773,7 +3856,17 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
 dependencies = [
- "phf_shared",
+ "phf_shared 0.12.1",
 ]
 [[package]]
 name = "phf"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
 dependencies = [
 "phf_shared 0.13.1",
 "serde",
 ]
 [[package]]
@@ -3785,6 +3878,15 @@ dependencies = [
 "siphasher",
 ]
 [[package]]
 name = "phf_shared"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
 dependencies = [
 "siphasher",
 ]
 [[package]]
 name = "pin-project"
 version = "1.1.11"
@@ -3823,6 +3925,38 @@ version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
 [[package]]
 name = "postgres-protocol"
 version = "0.6.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491"
 dependencies = [
 "base64",
 "byteorder",
 "bytes",
 "fallible-iterator",
 "hmac",
 "md-5",
 "memchr",
 "rand 0.9.2",
 "sha2",
 "stringprep",
 ]
 [[package]]
 name = "postgres-types"
 version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20"
 dependencies = [
 "bytes",
 "chrono",
 "fallible-iterator",
 "postgres-protocol",
 "serde_core",
 "serde_json",
 ]
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -4769,6 +4903,17 @@ dependencies = [
 "tracing",
 ]
 [[package]]
 name = "stringprep"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
 dependencies = [
 "unicode-bidi",
 "unicode-normalization",
 "unicode-properties",
 ]
 [[package]]
 name = "subsecond"
 version = "0.7.3"
@@ -4904,7 +5049,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
 dependencies = [
 "byteorder",
 "integer-encoding",
- "ordered-float",
+ "ordered-float 2.10.1",
 ]
 [[package]]
@@ -5000,6 +5145,32 @@ dependencies = [
 "syn",
 ]
 [[package]]
 name = "tokio-postgres"
 version = "0.7.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42"
 dependencies = [
 "async-trait",
 "byteorder",
 "bytes",
 "fallible-iterator",
 "futures-channel",
 "futures-util",
 "log",
 "parking_lot",
 "percent-encoding",
 "phf 0.13.1",
 "pin-project-lite",
 "postgres-protocol",
 "postgres-types",
 "rand 0.9.2",
 "socket2 0.6.3",
 "tokio",
 "tokio-util",
 "whoami",
 ]
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"
@@ -5356,12 +5527,33 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
 [[package]]
 name = "unicode-bidi"
 version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 [[package]]
 name = "unicode-normalization"
 version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
 dependencies = [
 "tinyvec",
 ]
 [[package]]
 name = "unicode-properties"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
 [[package]]
 name = "unicode-segmentation"
 version = "1.13.2"
@@ -5437,6 +5629,7 @@ dependencies = [
 "axum",
 "bytes",
 "chrono",
 "instant-distance",
 "object_store",
 "parquet",
 "serde",
@@ -5500,6 +5693,15 @@ version = "0.11.1+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 [[package]]
 name = "wasi"
 version = "0.14.7+wasi-0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
 dependencies = [
 "wasip2",
 ]
 [[package]]
 name = "wasip2"
 version = "1.0.2+wasi-0.2.9"
@@ -5518,6 +5720,15 @@ dependencies = [
 "wit-bindgen",
 ]
 [[package]]
 name = "wasite"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42"
 dependencies = [
 "wasi 0.14.7+wasi-0.2.4",
 ]
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.114"
@@ -5659,6 +5870,19 @@ version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
 [[package]]
 name = "whoami"
 version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d"
 dependencies = [
 "libc",
 "libredox",
 "objc2-system-configuration",
 "wasite",
 "web-sys",
 ]
 [[package]]
 name = "winapi-util"
 version = "0.1.11"
@@ -44,3 +44,5 @@ toml = "0.8"
 csv = "1"
 lopdf = "0.35"
 encoding_rs = "0.8"
 instant-distance = "0.6"
 tokio-postgres = { version = "0.7", features = ["with-serde_json-1", "with-chrono-0_4"] }
@@ -0,0 +1,183 @@
 # PRD: Observer Knowledge Base
 **Status:** Active — extraction pipeline proven, needs multi-pass processing via LLM Team modes
 **Created:** 2026-03-28
 **Owner:** J
 **Branch:** `observer-kb`
 **Parent:** Lakehouse core (crates/)
 ---
 ## Problem
 AI conversations generate massive amounts of knowledge that's lost to context windows, hallucination, and scope drift. Current approach (single-pass extraction into categories) produces disorganized, hard-to-validate output. Reading raw pages is not useful — the system needs to process, clean, structure, and curate knowledge like an editorial team.
 ## Vision
 A publishing pipeline that takes raw AI conversation exports and produces well-curated, validated knowledge documents. The output should be something you hand to a team — organized, filtered, and clearly presented. Questions anchor the content, with branching connections showing what was asked, what was answered, and how reasoning was directed.
 ---
 ## Architecture
 ### Multi-Pass Processing via LLM Team Modes
 The key insight: devop.live already has 20 analysis modes that process content through structured workflows. The Observer KB should route uploaded data through these modes automatically.
 ```
 Raw PDF Upload
    │
    ▼ Mode: EXTRACT
    Structured facts, entities, relationships
    │
    ▼ Mode: VALIDATOR
    Fact-check claims, score 1-10 confidence
    │
    ▼ Mode: HALLUCINATION
    Verify each claim, flag likely hallucinations
    │
    ▼ Mode: CONSENSUS
    Multiple passes iterate until extraction converges
    │
    ▼ Mode: REDTEAM
    Find flaws in knowledge, patch them
    │
    ▼ Mode: PIPELINE (custom)
    Clean → Q&A structure → topic grouping → value ranking
    │
    ▼ RENDER
    Well-curated document: questions as anchors, branching connections,
    what was asked → answered → directed
 ```
 ### Integration with devop.live (LLM Team)
 The Observer KB calls the LLM Team API to run modes against extracted content:
 | Mode | Purpose in KB |
 |---|---|
 | **extract** | Structured facts, entities, relationships from raw text |
 | **validator** | Fact-check extracted claims, confidence scores |
 | **hallucination** | Verify each factual claim independently |
 | **drift** | Process N times, find consistent vs uncertain content |
 | **redteam** | Find flaws in extracted knowledge, improve |
 | **consensus** | Iterate until multiple passes agree on correct extraction |
 | **pipeline** | Chain: clean → extract → validate → organize → render |
 | **research** | Generate follow-up questions, research in parallel |
 | **blindassembly** | Split content into parts, process independently, reassemble |
 ### Data Flow
 ```
 Upload Portal (Observer KB :3400)
    │
    ├── PDF → Lakehouse ingestd (text extraction, Parquet storage)
    │
    ▼
 Processing Pipeline
    │
    ├── Call devop.live API (:5000) with each mode
    ├── Each mode processes content through local Ollama models
    ├── Results stored back in Lakehouse (Parquet + catalog)
    │
    ▼
 Knowledge Store
    │
    ├── knowledge_facts: verified claims with confidence scores
    ├── knowledge_qa: question → answer pairs with context
    ├── knowledge_topics: auto-generated categories
    ├── knowledge_connections: how topics relate across conversations
    ├── knowledge_contradictions: flagged conflicts between conversations
    ├── knowledge_validated: claims that passed hallucination check
    │
    ▼
 Rendering
    │
    ├── Structured knowledge view (web UI)
    ├── Exportable curated document (PDF/markdown)
    └── Question-anchored navigation (Q → A → follow-up → related)
 ```
 ---
 ## Processing Pipeline Detail
 ### Pass 1: Clean + Structure
 - Strip PDF artifacts (headers, footers, page numbers)
 - Identify conversation turns (user/AI)
 - Fix encoding issues (ligatures, special chars)
 - Normalize formatting
 ### Pass 2: Q&A Extraction
 - Identify questions asked by user
 - Map each question to its answer
 - Track follow-up questions and how they branch
 - Build question→answer→follow-up chains
 ### Pass 3: Fact Extraction + Validation
 - Run EXTRACT mode: pull structured facts, entities, relationships
 - Run VALIDATOR mode: score each fact 1-10 confidence
 - Run HALLUCINATION mode: verify claims independently
 - Flag contradictions between conversations
 ### Pass 4: Organize + Curate
 - Run CONSENSUS mode: multiple passes converge on correct categorization
 - Auto-generate topic categories from content (not predefined)
 - Rank content by value (decisions > insights > discussion > noise)
 - Connect related content across conversations
 ### Pass 5: Render
 - Question-anchored view: each question is a node, answers branch from it
 - Topic-grouped view: knowledge organized by auto-generated categories
 - Timeline view: how thinking evolved across conversations
 - Export as clean markdown/PDF
 ---
 ## Current State (as of 2026-03-28)
 ### What's Built
 - PDF ingestion (8 PDFs, 477 pages extracted)
 - Single-pass knowledge extraction (496 topics, 300 decisions, 100 insights)
 - Semantic search via HNSW (27ms over 477 chunks)
 - RAG (question → search → LLM answer with sources)
 - 5-pass deep analysis (corpus overview, patterns, evolution, gaps, synthesis)
 - Observer KB web UI (knowledge-first, not page-first)
 - Upload portal with drag & drop
 ### What's Not Built Yet
 - Integration with devop.live LLM Team modes
 - Multi-pass processing pipeline (extract → validate → hallucination check)
 - Q&A chain extraction (question → answer → follow-up mapping)
 - Contradiction detection between conversations
 - Curated document export (PDF/markdown)
 - Question-anchored navigation
 - Auto-processing on upload (currently manual script)
 ### What Needs to Happen Next Session
 1. **Integrate with devop.live API** — call modes programmatically
 2. **Build the multi-pass pipeline** — extract → validate → hallucinate-check → organize
 3. **Q&A chain extraction** — questions as anchors, answers branching
 4. **Auto-process on upload** — no manual script, pipeline runs automatically
 5. **Render curated output** — readable, hand-it-to-someone quality
 ---
 ## Tech Stack
 - **Storage:** Lakehouse storaged (Parquet on object storage)
 - **Catalog:** Lakehouse catalogd (metadata, PII, lineage)
 - **Query:** Lakehouse queryd (DataFusion SQL, hot cache, paged results)
 - **Vectors:** Lakehouse vectord (HNSW search, RAG)
 - **AI Processing:** devop.live LLM Team API (20 modes, multiple models)
 - **AI Runtime:** Ollama (local models — qwen2.5, mistral, gemma2, nomic-embed-text)
 - **UI:** Pure HTML/JS on :3400
 ---
 ## Non-Goals (for now)
 - Real-time conversation ingestion (PDF exports only)
 - Multi-user collaboration
 - Cloud deployment
 - Editing extracted knowledge from UI (read-only for now)
@@ -0,0 +1,286 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta charset="UTF-8">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Observer KB</title>
 <style>
 :root{--bg:#0a0a0f;--s1:#12121a;--s2:#1a1a28;--b:#2a2a3a;--t:#e0e0e8;--d:#888898;--a:#6c5ce7;--ok:#00d2a0;--err:#ff6b6b;--warn:#ffa94d;--f:'JetBrains Mono','Fira Code',monospace}
 *{margin:0;padding:0;box-sizing:border-box}body{font-family:var(--f);background:var(--bg);color:var(--t);font-size:13px;min-height:100vh}
 .app{max-width:1200px;margin:0 auto;padding:20px}
 header{display:flex;align-items:center;gap:20px;padding:16px 0;border-bottom:1px solid var(--b);margin-bottom:20px}
 header h1{font-size:20px;color:var(--a)}
 .tabs{display:flex;gap:2px}.tab{background:none;border:none;color:var(--d);font-family:var(--f);font-size:12px;padding:10px 16px;cursor:pointer;border-bottom:2px solid transparent}
 .tab:hover{color:var(--t)}.tab.active{color:var(--a);border-bottom-color:var(--a)}
 .hdr-stats{margin-left:auto;font-size:11px;color:var(--d)}
 .btn{background:var(--a);color:white;border:none;padding:8px 16px;border-radius:6px;font-family:var(--f);font-size:12px;cursor:pointer;font-weight:600}
 .btn:hover{opacity:.9}.btn:disabled{opacity:.5}.btn-sm{padding:5px 10px;font-size:11px}
 .hidden{display:none}
 .stats-bar{display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:8px;margin-bottom:20px}
 .stat{background:var(--s1);border:1px solid var(--b);border-radius:8px;padding:12px;text-align:center}
 .stat-val{font-size:22px;font-weight:700;color:var(--a)}.stat-label{font-size:10px;color:var(--d);text-transform:uppercase;margin-top:2px}
 .section{margin-bottom:24px}.section h2{font-size:13px;color:var(--d);text-transform:uppercase;letter-spacing:.08em;margin-bottom:10px;border-bottom:1px solid var(--b);padding-bottom:6px}
 .entry{background:var(--s1);border:1px solid var(--b);border-radius:8px;padding:14px;margin-bottom:8px}
 .badge{font-size:9px;text-transform:uppercase;letter-spacing:.08em;padding:2px 8px;border-radius:3px;display:inline-block;margin-bottom:6px;font-weight:600}
 .badge-fact{background:var(--a);color:white}.badge-entity{background:var(--ok);color:white}
 .badge-decision{background:#e67e22;color:white}.badge-insight{background:#3498db;color:white}
 .badge-question{background:var(--warn);color:#000}.badge-verified{background:var(--ok);color:white}
 .entry-text{font-size:12px;line-height:1.6}.entry-src{font-size:10px;color:var(--d);margin-top:4px}
 .cat-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(250px,1fr));gap:8px;margin-bottom:16px}
 .cat-card{background:var(--s1);border:1px solid var(--b);border-radius:8px;padding:14px;cursor:pointer;border-left:3px solid var(--a);transition:all .15s}
 .cat-card:hover{border-color:var(--a)}.cat-title{font-weight:600;font-size:13px}.cat-count{font-size:10px;color:var(--d);margin-top:2px}
 .search-row{display:flex;gap:8px;margin-bottom:16px}
 .search-row input{flex:1;background:var(--s1);border:2px solid var(--b);color:var(--t);font-family:var(--f);font-size:14px;padding:12px;border-radius:8px;outline:none}
 .search-row input:focus{border-color:var(--a)}
 .ask-input{width:100%;background:var(--s1);border:2px solid var(--b);color:var(--t);font-family:var(--f);font-size:13px;padding:12px;border-radius:8px;resize:vertical;min-height:80px;outline:none;margin-bottom:8px}
 .ask-input:focus{border-color:var(--a)}
 .ai-answer{background:var(--s2);border:1px solid var(--b);border-radius:8px;padding:16px;margin-top:12px;line-height:1.7;white-space:pre-wrap}
 .ai-meta{font-size:10px;color:var(--d);margin-top:6px}
 .upload-zone{border:2px dashed var(--b);border-radius:12px;padding:36px;text-align:center;cursor:pointer;margin-bottom:16px}
 .upload-zone:hover{border-color:var(--a);background:rgba(108,92,231,.03)}
 .upload-zone h3{margin-bottom:4px}.upload-zone p{color:var(--d);font-size:11px}
 .file-item{display:flex;align-items:center;justify-content:space-between;padding:8px 12px;border-bottom:1px solid var(--b);font-size:12px}
 .done{color:var(--ok)}.failed{color:var(--err)}.uploading{color:var(--a)}
 .result{background:var(--s2);border:1px solid var(--b);border-radius:8px;padding:12px;margin-bottom:6px;border-left:3px solid var(--a)}
 .result-score{font-size:10px;color:var(--a);margin-bottom:2px}.result-text{font-size:12px;line-height:1.5}
 .empty{color:var(--d);text-align:center;padding:32px}
 </style>
 </head>
 <body>
 <div class="app">
 <header>
 <h1>OBSERVER KB</h1>
 <div class="tabs">
 <button class="tab active" onclick="showTab('knowledge')">Knowledge</button>
 <button class="tab" onclick="showTab('facts')">Facts &amp; Entities</button>
 <button class="tab" onclick="showTab('explore')">Search</button>
 <button class="tab" onclick="showTab('ask')">Ask AI</button>
 <button class="tab" onclick="showTab('upload')">Ingest</button>
 </div>
 <div class="hdr-stats" id="hdr-stats"></div>
 </header>
 <div id="tab-knowledge"></div>
 <div id="tab-facts" class="hidden"></div>
 <div id="tab-explore" class="hidden"></div>
 <div id="tab-ask" class="hidden"></div>
 <div id="tab-upload" class="hidden"></div>
 </div>
 <script>
 const API=`http://${location.hostname}:3100`;
 const $=id=>document.getElementById(id);
 async function q(sql){return(await(await fetch(API+'/query/sql',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({sql})})).json())}
 async function searchHNSW(query,k=10){try{return(await(await fetch(API+'/vectors/hnsw/search',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({index_name:'observer_kb',query,top_k:k})})).json()).results||[]}catch{return[]}}
 function showTab(name){
 ['knowledge','facts','explore','ask','upload'].forEach((t,i)=>{
 $('tab-'+t).classList.toggle('hidden',t!==name);
 document.querySelectorAll('.tab')[i].classList.toggle('active',t===name);
 });
 ({knowledge:loadKnowledge,facts:loadFacts,explore:loadExplore,ask:loadAsk,upload:loadUpload})[name]();
 }
 function makeEntry(type,text,source){
 const e=document.createElement('div');e.className='entry';
 const b=document.createElement('span');b.className='badge badge-'+type;b.textContent=type;
 const t=document.createElement('div');t.className='entry-text';t.textContent=text;
 const s=document.createElement('div');s.className='entry-src';s.textContent=source;
 e.append(b,t,s);return e;
 }
 // === KNOWLEDGE (categories + decisions + insights) ===
 async function loadKnowledge(){
 const el=$('tab-knowledge');el.textContent='';
 try{
 const[topics,decisions,insights,questions,extractions]=await Promise.all([
 q("SELECT topic,COUNT(*) as n,COUNT(DISTINCT conversation) as c FROM knowledge_topics GROUP BY topic ORDER BY n DESC LIMIT 15"),
 q("SELECT conversation,page,decision FROM knowledge_decisions LIMIT 12"),
 q("SELECT conversation,page,insight FROM knowledge_insights LIMIT 8"),
 q("SELECT COUNT(*) as n FROM knowledge_questions"),
 q("SELECT type,COUNT(*) as n FROM processed_extractions GROUP BY type"),
 ]);
 const factCount=extractions.rows.find(r=>r.type==='fact')?.n||0;
 const entityCount=extractions.rows.find(r=>r.type==='entity')?.n||0;
 // Stats
 const sb=document.createElement('div');sb.className='stats-bar';
 [{v:topics.row_count,l:'Categories'},{v:factCount,l:'Verified Facts'},{v:entityCount,l:'Entities'},{v:decisions.row_count,l:'Decisions'},{v:insights.row_count,l:'Insights'},{v:questions.rows[0]?.n||0,l:'Open Questions'}].forEach(s=>{
 const d=document.createElement('div');d.className='stat';
 const v=document.createElement('div');v.className='stat-val';v.textContent=s.v;
 const l=document.createElement('div');l.className='stat-label';l.textContent=s.l;
 d.append(v,l);sb.appendChild(d);
 });
 el.appendChild(sb);
 $('hdr-stats').textContent=factCount+' facts · '+entityCount+' entities · '+decisions.row_count+' decisions';
 // Categories
 const cs=document.createElement('div');cs.className='section';
 cs.insertAdjacentHTML('beforeend','<h2>Auto-Generated Categories</h2>');
 const cg=document.createElement('div');cg.className='cat-grid';
 topics.rows.forEach(t=>{
 const c=document.createElement('div');c.className='cat-card';
 const n=document.createElement('div');n.className='cat-title';n.textContent=t.topic;
 const m=document.createElement('div');m.className='cat-count';m.textContent=t.n+' mentions · '+t.c+' conversations';
 c.append(n,m);c.onclick=()=>drillTopic(t.topic);cg.appendChild(c);
 });
 cs.appendChild(cg);el.appendChild(cs);
 // Decisions
 const ds=document.createElement('div');ds.className='section';
 ds.insertAdjacentHTML('beforeend','<h2>Key Decisions</h2>');
 decisions.rows.forEach(d=>ds.appendChild(makeEntry('decision',d.decision,d.conversation+' p'+d.page)));
 el.appendChild(ds);
 // Insights
 const is=document.createElement('div');is.className='section';
 is.insertAdjacentHTML('beforeend','<h2>Key Insights</h2>');
 insights.rows.forEach(d=>is.appendChild(makeEntry('insight',d.insight,d.conversation+' p'+d.page)));
 el.appendChild(is);
 }catch(e){el.textContent='Error: '+e}
 }
 async function drillTopic(topic){
 const el=$('tab-knowledge');el.textContent='';
 const back=document.createElement('button');back.className='btn btn-sm';back.textContent='← Back';back.onclick=loadKnowledge;
 const h=document.createElement('h2');h.style.cssText='font-size:18px;margin:12px 0';h.textContent=topic;
 el.append(back,h);
 const results=await searchHNSW(topic,10);
 const s=document.createElement('div');s.className='section';
 s.insertAdjacentHTML('beforeend','<h2>Related Content</h2>');
 results.forEach(r=>{
 const e=document.createElement('div');e.className='result';
 const sc=document.createElement('div');sc.className='result-score';sc.textContent='['+r.score.toFixed(3)+'] '+r.doc_id;
 const t=document.createElement('div');t.className='result-text';t.textContent=r.chunk_text.substring(0,300);
 e.append(sc,t);s.appendChild(e);
 });
 el.appendChild(s);
 }
 // === FACTS & ENTITIES (from LLM Team extract mode) ===
 async function loadFacts(){
 const el=$('tab-facts');el.textContent='';
 try{
 const[facts,entities,verified]=await Promise.all([
 q("SELECT conversation,page,content FROM processed_extractions WHERE type='fact' ORDER BY conversation,page LIMIT 30"),
 q("SELECT conversation,content,COUNT(*) as n FROM processed_extractions WHERE type='entity' GROUP BY conversation,content ORDER BY n DESC LIMIT 30"),
 q("SELECT conversation,page,content FROM processed_validated LIMIT 10"),
 ]);
 const fs=document.createElement('div');fs.className='section';
 fs.insertAdjacentHTML('beforeend','<h2>Extracted Facts ('+facts.row_count+' total — via LLM Team Extract Mode)</h2>');
 facts.rows.forEach(r=>fs.appendChild(makeEntry('fact',r.content,r.conversation+' p'+r.page)));
 el.appendChild(fs);
 const es=document.createElement('div');es.className='section';
 es.insertAdjacentHTML('beforeend','<h2>Entities ('+entities.row_count+' unique)</h2>');
 const eg=document.createElement('div');eg.style.cssText='display:flex;flex-wrap:wrap;gap:6px';
 entities.rows.forEach(r=>{
 const chip=document.createElement('span');
 chip.style.cssText='background:var(--s2);border:1px solid var(--b);padding:4px 10px;border-radius:4px;font-size:11px;cursor:pointer';
 chip.textContent=r.content+(r.n>1?' ('+r.n+')':'');
 chip.onclick=()=>{showTab('explore');setTimeout(()=>{document.getElementById('searchInput').value=r.content;doSearch()},100)};
 eg.appendChild(chip);
 });
 es.appendChild(eg);el.appendChild(es);
 if(verified.rows.length){
 const vs=document.createElement('div');vs.className='section';
 vs.insertAdjacentHTML('beforeend','<h2>Verified by Secondary Model</h2>');
 verified.rows.forEach(r=>vs.appendChild(makeEntry('verified',r.content,r.conversation+' p'+r.page)));
 el.appendChild(vs);
 }
 }catch(e){el.textContent='Error: '+e}
 }
 // === SEARCH ===
 function loadExplore(){
 const el=$('tab-explore');el.textContent='';
 const row=document.createElement('div');row.className='search-row';
 const input=document.createElement('input');input.id='searchInput';input.placeholder='Search by meaning across all knowledge...';
 const btn=document.createElement('button');btn.className='btn';btn.textContent='Search';
 btn.onclick=doSearch;input.onkeydown=e=>{if(e.key==='Enter')doSearch()};
 row.append(input,btn);
 const res=document.createElement('div');res.id='searchResults';
 const hint=document.createElement('div');hint.className='empty';hint.textContent='Try: "object storage architecture", "data ingestion trade-offs", "CRM options"';
 el.append(row,res,hint);
 }
 async function doSearch(){
 const query=document.getElementById('searchInput').value;if(!query)return;
 const el=$('searchResults');el.textContent='searching...';
 const results=await searchHNSW(query,15);
 el.textContent='';
 if(!results.length){el.textContent='No results';return}
 results.forEach(r=>{
 const e=document.createElement('div');e.className='result';
 const s=document.createElement('div');s.className='result-score';s.textContent='['+r.score.toFixed(3)+'] '+r.doc_id.replace(/_/g,' ');
 const t=document.createElement('div');t.className='result-text';t.textContent=r.chunk_text.substring(0,400);
 e.append(s,t);el.appendChild(e);
 });
 }
 // === ASK AI ===
 function loadAsk(){
 const el=$('tab-ask');el.textContent='';
 const ta=document.createElement('textarea');ta.className='ask-input';ta.id='askInput';
 ta.placeholder='Ask a question about your knowledge base...\n\nExamples:\n- What architecture decisions were made about Iceberg?\n- Compare Spark vs Flink approaches\n- What approaches were abandoned and why?';
 const btn=document.createElement('button');btn.className='btn';btn.id='askBtn';btn.textContent='Ask AI';
 btn.onclick=doAsk;
 const resp=document.createElement('div');resp.id='askResp';
 el.append(ta,btn,resp);
 }
 async function doAsk(){
 const question=$('askInput').value;if(!question)return;
 $('askBtn').disabled=true;$('askBtn').textContent='thinking...';
 const el=$('askResp');el.textContent='searching knowledge...';
 try{
 const r=await(await fetch(API+'/vectors/rag',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({index_name:'observer_kb',question,top_k:5})})).json();
 el.textContent='';
 const ans=document.createElement('div');ans.className='ai-answer';ans.textContent=r.answer;
 const meta=document.createElement('div');meta.className='ai-meta';meta.textContent='Model: '+r.model+(r.tokens_generated?' · '+r.tokens_generated+' tokens':'');
 el.append(ans,meta);
 if(r.sources?.length){
 const sh=document.createElement('h2');sh.style.cssText='font-size:11px;color:var(--d);margin-top:14px';sh.textContent='SOURCES';el.appendChild(sh);
 r.sources.forEach(s=>{
 const e=document.createElement('div');e.className='result';e.style.marginTop='4px';
 const sc=document.createElement('div');sc.className='result-score';sc.textContent='['+s.score.toFixed(3)+'] '+s.doc_id;
 const t=document.createElement('div');t.className='result-text';t.style.fontSize='11px';t.textContent=s.chunk_text.substring(0,200);
 e.append(sc,t);el.appendChild(e);
 });
 }
 }catch(e){el.textContent='Error: '+e}
 $('askBtn').disabled=false;$('askBtn').textContent='Ask AI';
 }
 // === UPLOAD ===
 function loadUpload(){
 const el=$('tab-upload');el.textContent='';
 const zone=document.createElement('div');zone.className='upload-zone';
 zone.onclick=()=>{const i=document.createElement('input');i.type='file';i.multiple=true;i.accept='.pdf,.txt,.csv,.json';i.onchange=e=>[...e.target.files].forEach(uploadFile);i.click()};
 zone.ondragover=e=>{e.preventDefault();zone.style.borderColor='var(--a)'};
 zone.ondragleave=()=>zone.style.borderColor='';
 zone.ondrop=e=>{e.preventDefault();zone.style.borderColor='';[...e.dataTransfer.files].forEach(uploadFile)};
 const h=document.createElement('h3');h.textContent='Drop files to ingest';
 const p=document.createElement('p');p.textContent='PDF, CSV, JSON, Text — auto-extracted and indexed';
 zone.append(h,p);
 const list=document.createElement('div');list.id='upList';
 el.append(zone,list);
 }
 function uploadFile(file){
 const item=document.createElement('div');item.className='file-item';
 const name=document.createElement('span');name.textContent=file.name;
 const status=document.createElement('span');status.className='uploading';status.textContent='uploading...';
 item.append(name,status);$('upList').prepend(item);
 const fd=new FormData();fd.append('file',file);
 const xhr=new XMLHttpRequest();xhr.open('POST',API+'/ingest/file');
 xhr.upload.onprogress=e=>{if(e.lengthComputable)status.textContent=Math.round(e.loaded/e.total*100)+'%'};
 xhr.onload=()=>{status.className=xhr.status<300?'done':'failed';status.textContent=xhr.status<300?'ingested':'failed'};
 xhr.onerror=()=>{status.className='failed';status.textContent='error'};
 xhr.send(fd);
 }
 loadKnowledge();
 </script>
 </body>
 </html>
@@ -1,4 +1,7 @@
-use shared::types::{DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint};
+use shared::types::{
    DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint,
    ColumnMeta, Lineage, FreshnessContract, Sensitivity,
 };
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
@@ -6,6 +9,19 @@ use tokio::sync::RwLock;
 use storaged::ops;
 use object_store::ObjectStore;
 /// Partial metadata update — only set fields are applied.
 #[derive(Debug, Clone, Default, serde::Deserialize)]
 pub struct MetadataUpdate {
    pub description: Option<String>,
    pub owner: Option<String>,
    pub sensitivity: Option<Sensitivity>,
    pub tags: Option<Vec<String>>,
    pub columns: Option<Vec<ColumnMeta>>,
    pub lineage: Option<Lineage>,
    pub freshness: Option<FreshnessContract>,
    pub row_count: Option<u64>,
 }
 const MANIFEST_PREFIX: &str = "_catalog/manifests";
 /// In-memory dataset registry backed by manifest persistence in object storage.
@@ -54,6 +70,14 @@ impl Registry {
            objects,
            created_at: now,
            updated_at: now,
            description: String::new(),
            owner: String::new(),
            sensitivity: None,
            columns: vec![],
            lineage: None,
            freshness: None,
            tags: vec![],
            row_count: None,
        };
        // Write-ahead: persist before in-memory update
@@ -68,6 +92,36 @@ impl Registry {
        Ok(manifest)
    }
    /// Update metadata on an existing dataset (owner, description, tags, sensitivity, etc.)
    pub async fn update_metadata(
        &self,
        name: &str,
        updates: MetadataUpdate,
    ) -> Result<DatasetManifest, String> {
        let mut datasets = self.datasets.write().await;
        let manifest = datasets.values_mut()
            .find(|d| d.name == name)
            .ok_or_else(|| format!("dataset not found: {name}"))?;
        if let Some(desc) = updates.description { manifest.description = desc; }
        if let Some(owner) = updates.owner { manifest.owner = owner; }
        if let Some(sens) = updates.sensitivity { manifest.sensitivity = Some(sens); }
        if let Some(tags) = updates.tags { manifest.tags = tags; }
        if let Some(cols) = updates.columns { manifest.columns = cols; }
        if let Some(lineage) = updates.lineage { manifest.lineage = Some(lineage); }
        if let Some(freshness) = updates.freshness { manifest.freshness = Some(freshness); }
        if let Some(count) = updates.row_count { manifest.row_count = Some(count); }
        manifest.updated_at = chrono::Utc::now();
        // Persist
        let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
        let json = serde_json::to_vec_pretty(manifest).map_err(|e| e.to_string())?;
        ops::put(&self.store, &manifest_key, json.into()).await?;
        let result = manifest.clone();
        Ok(result)
    }
    /// Get a dataset by ID.
    pub async fn get(&self, id: &DatasetId) -> Option<DatasetManifest> {
        let datasets = self.datasets.read().await;
@@ -18,6 +18,7 @@ pub fn router(registry: Registry) -> Router {
        .route("/datasets", get(list_datasets))
        .route("/datasets/{id}", get(get_dataset))
        .route("/datasets/by-name/{name}", get(get_dataset_by_name))
        .route("/datasets/by-name/{name}/metadata", post(update_metadata))
        .with_state(registry)
 }
@@ -47,6 +48,15 @@ struct DatasetResponse {
    objects: Vec<ObjectRefResponse>,
    created_at: String,
    updated_at: String,
    // Rich metadata
    description: String,
    owner: String,
    sensitivity: Option<shared::types::Sensitivity>,
    columns: Vec<shared::types::ColumnMeta>,
    lineage: Option<shared::types::Lineage>,
    freshness: Option<shared::types::FreshnessContract>,
    tags: Vec<String>,
    row_count: Option<u64>,
 }
 #[derive(Serialize)]
@@ -71,6 +81,14 @@ impl From<&shared::types::DatasetManifest> for DatasetResponse {
            }).collect(),
            created_at: m.created_at.to_rfc3339(),
            updated_at: m.updated_at.to_rfc3339(),
            description: m.description.clone(),
            owner: m.owner.clone(),
            sensitivity: m.sensitivity.clone(),
            columns: m.columns.clone(),
            lineage: m.lineage.clone(),
            freshness: m.freshness.clone(),
            tags: m.tags.clone(),
            row_count: m.row_count,
        }
    }
 }
@@ -123,3 +141,14 @@ async fn get_dataset_by_name(
        None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {name}"))),
    }
 }
 async fn update_metadata(
    State(registry): State<Registry>,
    Path(name): Path<String>,
    Json(updates): Json<crate::registry::MetadataUpdate>,
 ) -> impl IntoResponse {
    match registry.update_metadata(&name, updates).await {
        Ok(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
    }
 }
@@ -26,3 +26,5 @@ opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 opentelemetry-stdout = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 arrow = { workspace = true }
 chrono = { workspace = true }
@@ -0,0 +1,155 @@
 /// Access control: field-level sensitivity enforcement, column masking, query audit.
 /// Evaluates policies at query time — not at storage level.
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use shared::types::Sensitivity;
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 /// An agent's role determines what sensitivity levels they can see.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AgentRole {
    pub agent_name: String,
    pub role: String,                        // "recruiter", "admin", "analyst", "agent"
    pub allowed_sensitivity: Vec<Sensitivity>, // what they can see unmasked
 }
 /// A query audit entry.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct QueryAudit {
    pub id: String,
    pub agent: String,
    pub sql: String,
    pub datasets_accessed: Vec<String>,
    pub pii_fields_accessed: Vec<String>,
    pub timestamp: DateTime<Utc>,
    pub row_count: usize,
    pub allowed: bool,
    pub masked_fields: Vec<String>,
 }
 /// Access control manager.
 #[derive(Clone)]
 pub struct AccessControl {
    roles: Arc<RwLock<HashMap<String, AgentRole>>>,
    audit_log: Arc<RwLock<Vec<QueryAudit>>>,
    enabled: bool,
 }
 impl AccessControl {
    pub fn new(enabled: bool) -> Self {
        let ac = Self {
            roles: Arc::new(RwLock::new(HashMap::new())),
            audit_log: Arc::new(RwLock::new(Vec::new())),
            enabled,
        };
        ac
    }
    /// Register default roles.
    pub async fn register_defaults(&self) {
        let defaults = vec![
            AgentRole {
                agent_name: "admin".into(),
                role: "admin".into(),
                allowed_sensitivity: vec![
                    Sensitivity::Public, Sensitivity::Internal,
                    Sensitivity::Pii, Sensitivity::Phi, Sensitivity::Financial,
                ],
            },
            AgentRole {
                agent_name: "recruiter".into(),
                role: "recruiter".into(),
                allowed_sensitivity: vec![
                    Sensitivity::Public, Sensitivity::Internal, Sensitivity::Pii,
                ],
            },
            AgentRole {
                agent_name: "analyst".into(),
                role: "analyst".into(),
                allowed_sensitivity: vec![
                    Sensitivity::Public, Sensitivity::Internal, Sensitivity::Financial,
                ],
            },
            AgentRole {
                agent_name: "agent".into(),
                role: "agent".into(),
                allowed_sensitivity: vec![
                    Sensitivity::Public, Sensitivity::Internal,
                ],
            },
        ];
        let mut roles = self.roles.write().await;
        for role in defaults {
            roles.insert(role.agent_name.clone(), role);
        }
    }
    /// Register or update an agent role.
    pub async fn set_role(&self, role: AgentRole) {
        self.roles.write().await.insert(role.agent_name.clone(), role);
    }
    /// Get an agent's role.
    pub async fn get_role(&self, agent: &str) -> Option<AgentRole> {
        self.roles.read().await.get(agent).cloned()
    }
    /// List all roles.
    pub async fn list_roles(&self) -> Vec<AgentRole> {
        self.roles.read().await.values().cloned().collect()
    }
    /// Check if an agent can see a field with given sensitivity.
    pub async fn can_access(&self, agent: &str, sensitivity: &Sensitivity) -> bool {
        if !self.enabled { return true; }
        match self.roles.read().await.get(agent) {
            Some(role) => role.allowed_sensitivity.contains(sensitivity),
            None => false, // unknown agent = no access
        }
    }
    /// Determine which fields should be masked for an agent.
    pub async fn masked_fields(
        &self,
        agent: &str,
        columns: &[shared::types::ColumnMeta],
    ) -> Vec<String> {
        if !self.enabled { return vec![]; }
        let role = match self.roles.read().await.get(agent) {
            Some(r) => r.clone(),
            None => return columns.iter().filter(|c| c.sensitivity.is_some()).map(|c| c.name.clone()).collect(),
        };
        columns.iter()
            .filter(|col| {
                if let Some(ref sens) = col.sensitivity {
                    !role.allowed_sensitivity.contains(sens)
                } else {
                    false
                }
            })
            .map(|col| col.name.clone())
            .collect()
    }
    /// Log a query for audit.
    pub async fn log_query(&self, audit: QueryAudit) {
        self.audit_log.write().await.push(audit);
    }
    /// Get recent audit entries.
    pub async fn recent_audit(&self, limit: usize) -> Vec<QueryAudit> {
        let log = self.audit_log.read().await;
        let start = log.len().saturating_sub(limit);
        log[start..].iter().rev().cloned().collect()
    }
    pub fn is_enabled(&self) -> bool {
        self.enabled
    }
 }
@@ -0,0 +1,62 @@
 use axum::{
    Json, Router,
    extract::{Query, State},
    http::StatusCode,
    response::IntoResponse,
    routing::{get, post},
 };
 use serde::Deserialize;
 use crate::access::{AccessControl, AgentRole};
 pub fn router(ac: AccessControl) -> Router {
    Router::new()
        .route("/roles", get(list_roles))
        .route("/roles", post(set_role))
        .route("/audit", get(query_audit))
        .route("/check", post(check_access))
        .with_state(ac)
 }
 async fn list_roles(State(ac): State<AccessControl>) -> impl IntoResponse {
    Json(ac.list_roles().await)
 }
 async fn set_role(
    State(ac): State<AccessControl>,
    Json(role): Json<AgentRole>,
 ) -> impl IntoResponse {
    let name = role.agent_name.clone();
    ac.set_role(role).await;
    (StatusCode::OK, format!("role set: {name}"))
 }
 #[derive(Deserialize)]
 struct AuditQuery {
    limit: Option<usize>,
 }
 async fn query_audit(
    State(ac): State<AccessControl>,
    Query(q): Query<AuditQuery>,
 ) -> impl IntoResponse {
    Json(ac.recent_audit(q.limit.unwrap_or(50)).await)
 }
 #[derive(Deserialize)]
 struct CheckRequest {
    agent: String,
    sensitivity: shared::types::Sensitivity,
 }
 async fn check_access(
    State(ac): State<AccessControl>,
    Json(req): Json<CheckRequest>,
 ) -> impl IntoResponse {
    let allowed = ac.can_access(&req.agent, &req.sensitivity).await;
    Json(serde_json::json!({
        "agent": req.agent,
        "sensitivity": req.sensitivity,
        "allowed": allowed,
    }))
 }
@@ -1,5 +1,8 @@
 mod access;
 mod access_service;
 mod auth;
 mod observability;
 mod tools;
 use axum::{Router, extract::DefaultBodyLimit, routing::get};
 use proto::lakehouse::catalog_service_server::CatalogServiceServer;
@@ -37,6 +40,10 @@ async fn main() {
    // Event journal — append-only mutation log (flush every 100 events)
    let journal = journald::journal::Journal::new(store.clone(), 100);
    // Access control
    let access = access::AccessControl::new(config.auth.enabled);
    access.register_defaults().await;
    // Workspace manager for agent-specific overlays
    let workspace_mgr = queryd::workspace::WorkspaceManager::new(store.clone());
    if let Err(e) = workspace_mgr.rebuild().await {
@@ -51,19 +58,34 @@ async fn main() {
        .route("/health", get(health))
        .nest("/storage", storaged::service::router(store.clone()))
        .nest("/catalog", catalogd::service::router(registry.clone()))
-        .nest("/query", queryd::service::router(engine))
+        .nest("/query", queryd::service::router(engine.clone()))
        .nest("/ai", aibridge::service::router(ai_client.clone()))
        .nest("/ingest", ingestd::service::router(ingestd::service::IngestState {
            store: store.clone(),
            registry: registry.clone(),
        }))
-        .nest("/vectors", vectord::service::router(vectord::service::VectorState {
+        .nest("/vectors", vectord::service::router({
            let index_reg = vectord::index_registry::IndexRegistry::new(store.clone());
            let _ = index_reg.rebuild().await;
            vectord::service::VectorState {
                store: store.clone(),
                ai_client: ai_client.clone(),
                job_tracker: vectord::jobs::JobTracker::new(),
                index_registry: index_reg,
                hnsw_store: vectord::hnsw::HnswStore::new(),
            }
        }))
        .nest("/workspaces", queryd::workspace_service::router(workspace_mgr))
-        .nest("/journal", journald::service::router(journal));
+        .nest("/journal", journald::service::router(journal))
        .nest("/access", access_service::router(access))
        .nest("/tools", tools::service::router({
            let tool_reg = tools::registry::ToolRegistry::new_with_defaults();
            tool_reg.register_defaults().await;
            tools::ToolState {
                registry: tool_reg,
                query_fn: tools::QueryExecutor::new(engine.clone()),
            }
        }));
    // Auth middleware (if enabled)
    if config.auth.enabled {
@@ -86,6 +108,13 @@ async fn main() {
            .allow_headers(Any))
        .layer(TraceLayer::new_for_http());
    // File watcher — auto-ingest from ./inbox
    let _watcher = ingestd::watcher::spawn_watcher(
        ingestd::watcher::WatchConfig::default(),
        store.clone(),
        registry.clone(),
    );
    // Start gRPC server on port+1
    let grpc_port = config.gateway.port + 1;
    let catalog_grpc = catalogd::grpc::CatalogGrpc::new(registry);
@@ -0,0 +1,47 @@
 pub mod registry;
 pub mod service;
 use queryd::context::QueryEngine;
 use arrow::json::writer::{JsonArray, Writer as JsonWriter};
 /// State for the tool system.
 #[derive(Clone)]
 pub struct ToolState {
    pub registry: registry::ToolRegistry,
    pub query_fn: QueryExecutor,
 }
 /// Wraps QueryEngine to provide a simple execute interface for tools.
 #[derive(Clone)]
 pub struct QueryExecutor {
    engine: QueryEngine,
 }
 impl QueryExecutor {
    pub fn new(engine: QueryEngine) -> Self {
        Self { engine }
    }
    /// Execute SQL and return (rows as JSON, row count).
    pub async fn execute(&self, sql: &str) -> Result<(serde_json::Value, usize), String> {
        let batches = self.engine.query(sql).await?;
        if batches.is_empty() {
            return Ok((serde_json::Value::Array(vec![]), 0));
        }
        let mut buf = Vec::new();
        let mut writer = JsonWriter::<_, JsonArray>::new(&mut buf);
        for batch in &batches {
            writer.write(batch).map_err(|e| format!("JSON write: {e}"))?;
        }
        writer.finish().map_err(|e| format!("JSON finish: {e}"))?;
        drop(writer);
        let rows: serde_json::Value = serde_json::from_slice(&buf)
            .map_err(|e| format!("JSON parse: {e}"))?;
        let count = rows.as_array().map(|a| a.len()).unwrap_or(0);
        Ok((rows, count))
    }
 }
@@ -0,0 +1,274 @@
 /// Tool Registry: named, governed business actions for AI agents.
 /// Instead of raw SQL, agents call validated tools with audit trails.
 ///
 /// Each tool has:
 /// - Name and description (for LLM tool-use)
 /// - Parameter schema (validated before execution)
 /// - Permission level (read / write / admin)
 /// - Audit logging (every invocation recorded)
 /// - Rate limiting (per agent)
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 /// Permission level for a tool.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
 pub enum Permission {
    Read,   // auto-approved, no side effects
    Write,  // modifies data, logged
    Admin,  // destructive, requires confirmation
 }
 /// Parameter definition for a tool.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ParamDef {
    pub name: String,
    pub param_type: String, // "string", "integer", "boolean", "float"
    pub required: bool,
    pub description: String,
    pub default: Option<serde_json::Value>,
 }
 /// Tool definition — what agents see and can call.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ToolDef {
    pub name: String,
    pub description: String,
    pub permission: Permission,
    pub parameters: Vec<ParamDef>,
    pub returns: String,          // description of return value
    pub sql_template: String,     // SQL with {param} placeholders
    pub category: String,         // "candidates", "placements", "analytics"
 }
 /// Audit log entry for a tool invocation.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ToolInvocation {
    pub id: String,
    pub tool_name: String,
    pub agent: String,
    pub params: serde_json::Value,
    pub permission: Permission,
    pub timestamp: DateTime<Utc>,
    pub success: bool,
    pub error: Option<String>,
    pub rows_returned: Option<usize>,
 }
 /// The registry — holds tool definitions and audit log.
 #[derive(Clone)]
 pub struct ToolRegistry {
    tools: Arc<RwLock<HashMap<String, ToolDef>>>,
    audit_log: Arc<RwLock<Vec<ToolInvocation>>>,
 }
 impl ToolRegistry {
    pub fn new() -> Self {
        let registry = Self {
            tools: Arc::new(RwLock::new(HashMap::new())),
            audit_log: Arc::new(RwLock::new(Vec::new())),
        };
        // Register built-in staffing tools
        tokio::task::block_in_place(|| {
            tokio::runtime::Handle::current().block_on(registry.register_defaults())
        });
        registry
    }
    pub fn new_with_defaults() -> Self {
        let registry = Self {
            tools: Arc::new(RwLock::new(HashMap::new())),
            audit_log: Arc::new(RwLock::new(Vec::new())),
        };
        registry
    }
    /// Register default staffing tools.
    pub async fn register_defaults(&self) {
        let tools = vec![
            ToolDef {
                name: "search_candidates".into(),
                description: "Search for candidates by skills, city, state, availability, and experience. Returns matching candidates with contact info.".into(),
                permission: Permission::Read,
                parameters: vec![
                    ParamDef { name: "skills".into(), param_type: "string".into(), required: false, description: "Comma-separated skills to match (uses LIKE)".into(), default: None },
                    ParamDef { name: "city".into(), param_type: "string".into(), required: false, description: "City name".into(), default: None },
                    ParamDef { name: "state".into(), param_type: "string".into(), required: false, description: "State abbreviation".into(), default: None },
                    ParamDef { name: "min_years".into(), param_type: "integer".into(), required: false, description: "Minimum years of experience".into(), default: Some(serde_json::json!(0)) },
                    ParamDef { name: "status".into(), param_type: "string".into(), required: false, description: "Candidate status (active, inactive, placed)".into(), default: Some(serde_json::json!("active")) },
                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
                ],
                returns: "List of candidates with id, name, phone, email, skills, experience".into(),
                sql_template: "SELECT candidate_id, first_name, last_name, phone, email, city, state, zip, vertical, skills, years_experience FROM candidates WHERE 1=1 {skills_filter} {city_filter} {state_filter} {years_filter} {status_filter} ORDER BY years_experience DESC LIMIT {limit}".into(),
                category: "candidates".into(),
            },
            ToolDef {
                name: "get_candidate".into(),
                description: "Get full details for a specific candidate by ID.".into(),
                permission: Permission::Read,
                parameters: vec![
                    ParamDef { name: "candidate_id".into(), param_type: "string".into(), required: true, description: "Candidate ID (e.g. CAND-000001)".into(), default: None },
                ],
                returns: "Full candidate record".into(),
                sql_template: "SELECT * FROM candidates WHERE candidate_id = '{candidate_id}'".into(),
                category: "candidates".into(),
            },
            ToolDef {
                name: "revenue_by_client".into(),
                description: "Show total billed revenue, pay costs, and gross profit by client. Filter by date range.".into(),
                permission: Permission::Read,
                parameters: vec![
                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Top N clients".into(), default: Some(serde_json::json!(10)) },
                ],
                returns: "Client name, total billed, total paid, gross profit, timesheet count".into(),
                sql_template: "SELECT c.company_name, COUNT(*) as timesheets, ROUND(SUM(t.bill_total),2) as total_billed, ROUND(SUM(t.pay_total),2) as total_paid, ROUND(SUM(t.bill_total) - SUM(t.pay_total),2) as gross_profit FROM timesheets t JOIN clients c ON t.client_id = c.client_id WHERE t.approved = true GROUP BY c.company_name ORDER BY total_billed DESC LIMIT {limit}".into(),
                category: "analytics".into(),
            },
            ToolDef {
                name: "recruiter_performance".into(),
                description: "Show recruiter performance: placements, unique candidates, and total revenue generated.".into(),
                permission: Permission::Read,
                parameters: vec![
                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Top N recruiters".into(), default: Some(serde_json::json!(10)) },
                ],
                returns: "Recruiter name, placement count, unique candidates, total revenue".into(),
                sql_template: "SELECT p.recruiter, COUNT(DISTINCT p.placement_id) as placements, COUNT(DISTINCT p.candidate_id) as unique_candidates, ROUND(SUM(t.bill_total),2) as total_revenue FROM placements p JOIN timesheets t ON p.placement_id = t.placement_id GROUP BY p.recruiter ORDER BY total_revenue DESC LIMIT {limit}".into(),
                category: "analytics".into(),
            },
            ToolDef {
                name: "cold_leads".into(),
                description: "Find candidates who were called multiple times but never placed — potential lost opportunities.".into(),
                permission: Permission::Read,
                parameters: vec![
                    ParamDef { name: "min_calls".into(), param_type: "integer".into(), required: false, description: "Minimum call count".into(), default: Some(serde_json::json!(5)) },
                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
                ],
                returns: "Candidates with call counts who were never placed".into(),
                sql_template: "SELECT c.candidate_id, c.first_name, c.last_name, c.phone, c.vertical, cl.calls FROM candidates c JOIN (SELECT candidate_id, COUNT(*) as calls FROM call_log GROUP BY candidate_id HAVING COUNT(*) >= {min_calls}) cl ON c.candidate_id = cl.candidate_id WHERE c.candidate_id NOT IN (SELECT DISTINCT candidate_id FROM placements) ORDER BY cl.calls DESC LIMIT {limit}".into(),
                category: "analytics".into(),
            },
            ToolDef {
                name: "open_jobs".into(),
                description: "List open job orders with client, title, rates, and location.".into(),
                permission: Permission::Read,
                parameters: vec![
                    ParamDef { name: "vertical".into(), param_type: "string".into(), required: false, description: "Filter by vertical (IT, Healthcare, etc)".into(), default: None },
                    ParamDef { name: "city".into(), param_type: "string".into(), required: false, description: "Filter by city".into(), default: None },
                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
                ],
                returns: "Open job orders with details".into(),
                sql_template: "SELECT j.job_order_id, c.company_name, j.title, j.vertical, j.city, j.state, j.bill_rate, j.pay_rate, j.status FROM job_orders j JOIN clients c ON j.client_id = c.client_id WHERE j.status = 'open' {vertical_filter} {city_filter} ORDER BY j.bill_rate DESC LIMIT {limit}".into(),
                category: "jobs".into(),
            },
        ];
        let mut reg = self.tools.write().await;
        for tool in tools {
            reg.insert(tool.name.clone(), tool);
        }
    }
    /// Get tool definition (for LLM tool-use schema).
    pub async fn get_tool(&self, name: &str) -> Option<ToolDef> {
        self.tools.read().await.get(name).cloned()
    }
    /// List all tools (for LLM tool discovery).
    pub async fn list_tools(&self) -> Vec<ToolDef> {
        self.tools.read().await.values().cloned().collect()
    }
    /// Build SQL from tool parameters. Validates and sanitizes.
    pub fn build_sql(tool: &ToolDef, params: &serde_json::Value) -> Result<String, String> {
        let mut sql = tool.sql_template.clone();
        let params = params.as_object().ok_or("params must be an object")?;
        // Replace named parameters
        for param_def in &tool.parameters {
            let value = params.get(&param_def.name)
                .or(param_def.default.as_ref());
            match param_def.name.as_str() {
                // Handle filter parameters (conditional WHERE clauses)
                "skills" => {
                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
                        let filters: Vec<String> = v.split(',')
                            .map(|s| format!("skills LIKE '%{}%'", s.trim().replace('\'', "''")))
                            .collect();
                        sql = sql.replace("{skills_filter}", &format!("AND ({})", filters.join(" OR ")));
                    } else {
                        sql = sql.replace("{skills_filter}", "");
                    }
                }
                "city" => {
                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
                        sql = sql.replace("{city_filter}", &format!("AND city = '{}'", v.replace('\'', "''")));
                    } else {
                        sql = sql.replace("{city_filter}", "");
                    }
                }
                "state" => {
                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
                        sql = sql.replace("{state_filter}", &format!("AND state = '{}'", v.replace('\'', "''")));
                    } else {
                        sql = sql.replace("{state_filter}", "");
                    }
                }
                "vertical" => {
                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
                        sql = sql.replace("{vertical_filter}", &format!("AND j.vertical = '{}'", v.replace('\'', "''")));
                    } else {
                        sql = sql.replace("{vertical_filter}", "");
                    }
                }
                "status" => {
                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
                        sql = sql.replace("{status_filter}", &format!("AND status = '{}'", v.replace('\'', "''")));
                    } else {
                        sql = sql.replace("{status_filter}", "");
                    }
                }
                "min_years" => {
                    if let Some(v) = value.and_then(|v| v.as_i64()) {
                        sql = sql.replace("{years_filter}", &format!("AND years_experience >= {v}"));
                    } else {
                        sql = sql.replace("{years_filter}", "");
                    }
                }
                // Direct substitution for simple params
                _ => {
                    let placeholder = format!("{{{}}}", param_def.name);
                    if sql.contains(&placeholder) {
                        let val_str = match value {
                            Some(serde_json::Value::String(s)) => s.replace('\'', "''"),
                            Some(serde_json::Value::Number(n)) => n.to_string(),
                            Some(serde_json::Value::Bool(b)) => b.to_string(),
                            Some(v) => v.to_string(),
                            None if param_def.required => return Err(format!("missing required param: {}", param_def.name)),
                            None => continue,
                        };
                        sql = sql.replace(&placeholder, &val_str);
                    }
                }
            }
        }
        Ok(sql)
    }
    /// Log a tool invocation.
    pub async fn log_invocation(&self, inv: ToolInvocation) {
        self.audit_log.write().await.push(inv);
    }
    /// Get recent audit log.
    pub async fn recent_audit(&self, limit: usize) -> Vec<ToolInvocation> {
        let log = self.audit_log.read().await;
        let start = log.len().saturating_sub(limit);
        log[start..].iter().rev().cloned().collect()
    }
 }
@@ -0,0 +1,148 @@
 use axum::{
    Json, Router,
    extract::{Path, Query, State},
    http::StatusCode,
    response::IntoResponse,
    routing::{get, post},
 };
 use serde::Deserialize;
 use super::registry::{Permission, ToolInvocation, ToolRegistry};
 use crate::tools::ToolState;
 pub fn router(state: ToolState) -> Router {
    Router::new()
        .route("/", get(list_tools))
        .route("/{name}", get(get_tool))
        .route("/{name}/call", post(call_tool))
        .route("/audit", get(audit_log))
        .with_state(state)
 }
 /// List all available tools (for LLM tool-use discovery).
 async fn list_tools(State(state): State<ToolState>) -> impl IntoResponse {
    let tools = state.registry.list_tools().await;
    // Return in MCP-compatible format
    let tool_list: Vec<serde_json::Value> = tools.iter().map(|t| {
        serde_json::json!({
            "name": t.name,
            "description": t.description,
            "permission": t.permission,
            "category": t.category,
            "parameters": t.parameters.iter().map(|p| {
                serde_json::json!({
                    "name": p.name,
                    "type": p.param_type,
                    "required": p.required,
                    "description": p.description,
                    "default": p.default,
                })
            }).collect::<Vec<_>>(),
        })
    }).collect();
    Json(tool_list)
 }
 /// Get a specific tool definition.
 async fn get_tool(
    State(state): State<ToolState>,
    Path(name): Path<String>,
 ) -> impl IntoResponse {
    match state.registry.get_tool(&name).await {
        Some(tool) => Ok(Json(tool)),
        None => Err((StatusCode::NOT_FOUND, format!("tool not found: {name}"))),
    }
 }
 /// Call a tool — validate params, execute SQL, log invocation.
 #[derive(Deserialize)]
 struct CallRequest {
    params: serde_json::Value,
    agent: String,
 }
 async fn call_tool(
    State(state): State<ToolState>,
    Path(name): Path<String>,
    Json(req): Json<CallRequest>,
 ) -> impl IntoResponse {
    let tool = match state.registry.get_tool(&name).await {
        Some(t) => t,
        None => return Err((StatusCode::NOT_FOUND, format!("tool not found: {name}"))),
    };
    tracing::info!("tool call: {} by agent '{}' with {:?}", name, req.agent, req.params);
    // Build SQL from params
    let sql = match ToolRegistry::build_sql(&tool, &req.params) {
        Ok(s) => s,
        Err(e) => {
            state.registry.log_invocation(ToolInvocation {
                id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
                tool_name: name.clone(),
                agent: req.agent.clone(),
                params: req.params.clone(),
                permission: tool.permission.clone(),
                timestamp: chrono::Utc::now(),
                success: false,
                error: Some(e.clone()),
                rows_returned: None,
            }).await;
            return Err((StatusCode::BAD_REQUEST, e));
        }
    };
    // Execute via query engine
    let result = state.query_fn.execute(&sql).await;
    match result {
        Ok((rows, row_count)) => {
            state.registry.log_invocation(ToolInvocation {
                id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
                tool_name: name.clone(),
                agent: req.agent,
                params: req.params,
                permission: tool.permission,
                timestamp: chrono::Utc::now(),
                success: true,
                error: None,
                rows_returned: Some(row_count),
            }).await;
            Ok(Json(serde_json::json!({
                "tool": name,
                "rows": rows,
                "row_count": row_count,
                "sql": sql,
            })))
        }
        Err(e) => {
            state.registry.log_invocation(ToolInvocation {
                id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
                tool_name: name,
                agent: req.agent,
                params: req.params,
                permission: tool.permission,
                timestamp: chrono::Utc::now(),
                success: false,
                error: Some(e.clone()),
                rows_returned: None,
            }).await;
            Err((StatusCode::INTERNAL_SERVER_ERROR, e))
        }
    }
 }
 #[derive(Deserialize)]
 struct AuditQuery {
    limit: Option<usize>,
 }
 async fn audit_log(
    State(state): State<ToolState>,
    Query(q): Query<AuditQuery>,
 ) -> impl IntoResponse {
    let log = state.registry.recent_audit(q.limit.unwrap_or(50)).await;
    Json(log)
 }
@@ -20,3 +20,4 @@ sha2 = { workspace = true }
 csv = { workspace = true }
 chrono = { workspace = true }
 object_store = { workspace = true }
 tokio-postgres = { workspace = true }
@@ -0,0 +1,235 @@
 /// ChatGPT PDF conversation parser.
 /// Extracts structured conversation turns from ChatGPT export PDFs.
 /// Handles: user/assistant turns, timestamps, conversation titles.
 use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
 use arrow::datatypes::{DataType, Field, Schema};
 use std::sync::Arc;
 /// A single conversation turn.
 #[derive(Debug, Clone)]
 pub struct ConversationTurn {
    pub source_file: String,
    pub conversation_title: String,
    pub turn_number: i32,
    pub role: String,        // "user" or "assistant"
    pub content: String,
    pub page_number: i32,
 }
 /// Parse a ChatGPT PDF into structured conversation turns.
 pub fn parse_chatgpt_pdf(content: &[u8], filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
    let doc = lopdf::Document::load_mem(content)
        .map_err(|e| format!("PDF load error: {e}"))?;
    // Extract all text
    let pages = doc.get_pages();
    let mut all_text = String::new();
    let mut page_texts: Vec<(i32, String)> = Vec::new();
    for (&page_num, _) in pages.iter() {
        let text = doc.extract_text(&[page_num]).unwrap_or_default();
        let text = text.trim().to_string();
        if !text.is_empty() {
            page_texts.push((page_num as i32, text.clone()));
            all_text.push_str(&text);
            all_text.push('\n');
        }
    }
    if all_text.trim().is_empty() {
        return Err("PDF contains no extractable text".into());
    }
    // Try to parse as ChatGPT conversation format
    let turns = extract_turns(&all_text, filename, &page_texts);
    if turns.is_empty() {
        // Fall back to page-based extraction if we can't detect conversation structure
        return parse_as_raw_pages(&page_texts, filename);
    }
    tracing::info!("parsed ChatGPT PDF '{}': {} turns", filename, turns.len());
    turns_to_batch(&turns)
 }
 /// Extract conversation turns from ChatGPT PDF text.
 /// ChatGPT PDFs typically have patterns like:
 /// - "You said:" / "ChatGPT said:" (newer format)
 /// - "User:" / "Assistant:"
 /// - Or just alternating blocks separated by formatting
 fn extract_turns(text: &str, filename: &str, page_texts: &[(i32, String)]) -> Vec<ConversationTurn> {
    let mut turns = Vec::new();
    let mut turn_num = 0;
    // Try to extract conversation title from first line
    let title = text.lines().next()
        .map(|l| l.trim().to_string())
        .filter(|l| l.len() < 200 && !l.is_empty())
        .unwrap_or_else(|| filename.replace(".pdf", ""));
    // Strategy 1: Look for "You said:" / "ChatGPT said:" markers (common format)
    let you_markers = ["You said:", "You:", "User:", "Human:"];
    let ai_markers = ["ChatGPT said:", "ChatGPT:", "Assistant:", "AI:"];
    let has_markers = you_markers.iter().any(|m| text.contains(m))
        && ai_markers.iter().any(|m| text.contains(m));
    if has_markers {
        let lines: Vec<&str> = text.lines().collect();
        let mut current_role = String::new();
        let mut current_content = String::new();
        let mut current_page = 1i32;
        for line in &lines {
            let trimmed = line.trim();
            // Check for role markers
            let mut found_marker = false;
            for marker in &you_markers {
                if trimmed.starts_with(marker) || trimmed == *marker {
                    // Save previous turn
                    if !current_content.trim().is_empty() && !current_role.is_empty() {
                        turn_num += 1;
                        turns.push(ConversationTurn {
                            source_file: filename.to_string(),
                            conversation_title: title.clone(),
                            turn_number: turn_num,
                            role: current_role.clone(),
                            content: current_content.trim().to_string(),
                            page_number: current_page,
                        });
                    }
                    current_role = "user".to_string();
                    current_content = trimmed.strip_prefix(marker).unwrap_or("").trim().to_string();
                    current_content.push('\n');
                    found_marker = true;
                    break;
                }
            }
            if !found_marker {
                for marker in &ai_markers {
                    if trimmed.starts_with(marker) || trimmed == *marker {
                        if !current_content.trim().is_empty() && !current_role.is_empty() {
                            turn_num += 1;
                            turns.push(ConversationTurn {
                                source_file: filename.to_string(),
                                conversation_title: title.clone(),
                                turn_number: turn_num,
                                role: current_role.clone(),
                                content: current_content.trim().to_string(),
                                page_number: current_page,
                            });
                        }
                        current_role = "assistant".to_string();
                        current_content = trimmed.strip_prefix(marker).unwrap_or("").trim().to_string();
                        current_content.push('\n');
                        found_marker = true;
                        break;
                    }
                }
            }
            if !found_marker && !current_role.is_empty() {
                current_content.push_str(trimmed);
                current_content.push('\n');
            }
        }
        // Save last turn
        if !current_content.trim().is_empty() && !current_role.is_empty() {
            turn_num += 1;
            turns.push(ConversationTurn {
                source_file: filename.to_string(),
                conversation_title: title.clone(),
                turn_number: turn_num,
                role: current_role,
                content: current_content.trim().to_string(),
                page_number: current_page,
            });
        }
        return turns;
    }
    // Strategy 2: Split by paragraph blocks, alternate user/assistant
    // This is a heuristic fallback for PDFs without clear markers
    let paragraphs: Vec<&str> = text.split("\n\n")
        .map(|p| p.trim())
        .filter(|p| p.len() > 20) // skip short fragments
        .collect();
    if paragraphs.len() >= 2 {
        for (i, para) in paragraphs.iter().enumerate() {
            turn_num += 1;
            turns.push(ConversationTurn {
                source_file: filename.to_string(),
                conversation_title: title.clone(),
                turn_number: turn_num,
                role: if i % 2 == 0 { "user" } else { "assistant" }.to_string(),
                content: para.to_string(),
                page_number: 1,
            });
        }
    }
    turns
 }
 /// Fallback: parse as raw pages with full text.
 fn parse_as_raw_pages(page_texts: &[(i32, String)], filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
    let schema = Arc::new(Schema::new(vec![
        Field::new("source_file", DataType::Utf8, false),
        Field::new("conversation_title", DataType::Utf8, false),
        Field::new("turn_number", DataType::Int32, false),
        Field::new("role", DataType::Utf8, false),
        Field::new("content", DataType::Utf8, false),
        Field::new("page_number", DataType::Int32, false),
    ]));
    let sources: Vec<&str> = page_texts.iter().map(|_| filename).collect();
    let titles: Vec<String> = page_texts.iter().map(|_| filename.replace(".pdf", "")).collect();
    let turn_nums: Vec<i32> = (1..=page_texts.len() as i32).collect();
    let roles: Vec<&str> = page_texts.iter().map(|_| "page").collect();
    let contents: Vec<&str> = page_texts.iter().map(|(_, t)| t.as_str()).collect();
    let pages: Vec<i32> = page_texts.iter().map(|(p, _)| *p).collect();
    let arrays: Vec<ArrayRef> = vec![
        Arc::new(StringArray::from(sources)),
        Arc::new(StringArray::from(titles.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
        Arc::new(Int32Array::from(turn_nums)),
        Arc::new(StringArray::from(roles)),
        Arc::new(StringArray::from(contents)),
        Arc::new(Int32Array::from(pages)),
    ];
    let batch = RecordBatch::try_new(schema.clone(), arrays)
        .map_err(|e| format!("RecordBatch error: {e}"))?;
    Ok((schema, vec![batch]))
 }
 fn turns_to_batch(turns: &[ConversationTurn]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
    let schema = Arc::new(Schema::new(vec![
        Field::new("source_file", DataType::Utf8, false),
        Field::new("conversation_title", DataType::Utf8, false),
        Field::new("turn_number", DataType::Int32, false),
        Field::new("role", DataType::Utf8, false),
        Field::new("content", DataType::Utf8, false),
        Field::new("page_number", DataType::Int32, false),
    ]));
    let arrays: Vec<ArrayRef> = vec![
        Arc::new(StringArray::from(turns.iter().map(|t| t.source_file.as_str()).collect::<Vec<_>>())),
        Arc::new(StringArray::from(turns.iter().map(|t| t.conversation_title.as_str()).collect::<Vec<_>>())),
        Arc::new(Int32Array::from(turns.iter().map(|t| t.turn_number).collect::<Vec<_>>())),
        Arc::new(StringArray::from(turns.iter().map(|t| t.role.as_str()).collect::<Vec<_>>())),
        Arc::new(StringArray::from(turns.iter().map(|t| t.content.as_str()).collect::<Vec<_>>())),
        Arc::new(Int32Array::from(turns.iter().map(|t| t.page_number).collect::<Vec<_>>())),
    ];
    let batch = RecordBatch::try_new(schema.clone(), arrays)
        .map_err(|e| format!("RecordBatch error: {e}"))?;
    Ok((schema, vec![batch]))
 }
@@ -0,0 +1,225 @@
 /// Database connector: pull tables from PostgreSQL directly into Parquet.
 /// Connects, reads schema, streams rows into Arrow RecordBatches, writes Parquet.
 /// No ORM, no schema definition — schema inferred from the database.
 use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, StringArray};
 use arrow::datatypes::{DataType, Field, Schema};
 use arrow::record_batch::RecordBatch;
 use std::sync::Arc;
 use tokio_postgres::{Client, NoTls, types::Type as PgType};
 /// Connection config for a database.
 #[derive(Debug, Clone, serde::Deserialize)]
 pub struct DbConfig {
    pub host: String,
    pub port: u16,
    pub database: String,
    pub user: String,
    pub password: String,
 }
 impl DbConfig {
    pub fn connection_string(&self) -> String {
        if self.password.is_empty() {
            format!(
                "host={} port={} dbname={} user={}",
                self.host, self.port, self.database, self.user
            )
        } else {
            format!(
                "host={} port={} dbname={} user={} password={}",
                self.host, self.port, self.database, self.user, self.password
            )
        }
    }
 }
 /// Result of a database table import.
 #[derive(Debug, Clone, serde::Serialize)]
 pub struct DbIngestResult {
    pub table_name: String,
    pub rows: usize,
    pub columns: usize,
    pub schema_detected: Vec<String>, // column name: type pairs
 }
 /// Connect to PostgreSQL and import a table as Arrow RecordBatches.
 pub async fn import_postgres_table(
    config: &DbConfig,
    table_name: &str,
    limit: Option<usize>,
 ) -> Result<(Arc<Schema>, Vec<RecordBatch>, DbIngestResult), String> {
    // Connect
    let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
        .await
        .map_err(|e| format!("postgres connect error: {e}"))?;
    // Spawn connection handler
    tokio::spawn(async move {
        if let Err(e) = connection.await {
            tracing::error!("postgres connection error: {e}");
        }
    });
    tracing::info!("connected to postgres {}:{}/{}", config.host, config.port, config.database);
    // Get column info
    let columns = get_table_columns(&client, table_name).await?;
    if columns.is_empty() {
        return Err(format!("table '{}' not found or has no columns", table_name));
    }
    let schema_detected: Vec<String> = columns.iter()
        .map(|(name, pg_type)| format!("{}: {}", name, pg_type_to_string(pg_type)))
        .collect();
    tracing::info!("table '{}': {} columns", table_name, columns.len());
    // Build Arrow schema
    let arrow_fields: Vec<Field> = columns.iter()
        .map(|(name, pg_type)| Field::new(name, pg_type_to_arrow(pg_type), true))
        .collect();
    let schema = Arc::new(Schema::new(arrow_fields));
    // Query rows
    let sql = match limit {
        Some(n) => format!("SELECT * FROM \"{}\" LIMIT {}", table_name, n),
        None => format!("SELECT * FROM \"{}\"", table_name),
    };
    let rows = client.query(&sql, &[])
        .await
        .map_err(|e| format!("query error: {e}"))?;
    let row_count = rows.len();
    tracing::info!("fetched {} rows from '{}'", row_count, table_name);
    if row_count == 0 {
        return Ok((schema.clone(), vec![], DbIngestResult {
            table_name: table_name.to_string(),
            rows: 0,
            columns: columns.len(),
            schema_detected,
        }));
    }
    // Convert to Arrow arrays
    let mut arrays: Vec<ArrayRef> = Vec::new();
    for (col_idx, (_, pg_type)) in columns.iter().enumerate() {
        let array = rows_to_arrow_column(&rows, col_idx, pg_type)?;
        arrays.push(array);
    }
    let batch = RecordBatch::try_new(schema.clone(), arrays)
        .map_err(|e| format!("RecordBatch error: {e}"))?;
    let result = DbIngestResult {
        table_name: table_name.to_string(),
        rows: row_count,
        columns: columns.len(),
        schema_detected,
    };
    Ok((schema, vec![batch], result))
 }
 /// List all tables in the database.
 pub async fn list_postgres_tables(config: &DbConfig) -> Result<Vec<String>, String> {
    let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
        .await
        .map_err(|e| format!("connect error: {e}"))?;
    tokio::spawn(async move { let _ = connection.await; });
    let rows = client.query(
        "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name",
        &[],
    ).await.map_err(|e| format!("query error: {e}"))?;
    Ok(rows.iter().map(|r| r.get::<_, String>(0)).collect())
 }
 /// Get column names and types for a table.
 async fn get_table_columns(client: &Client, table_name: &str) -> Result<Vec<(String, PgType)>, String> {
    // Use a dummy query to get column types
    let stmt = client.prepare(&format!("SELECT * FROM \"{}\" LIMIT 0", table_name))
        .await
        .map_err(|e| format!("prepare error for '{}': {e}", table_name))?;
    Ok(stmt.columns().iter()
        .map(|col| (col.name().to_string(), col.type_().clone()))
        .collect())
 }
 /// Convert a Postgres type to Arrow DataType.
 fn pg_type_to_arrow(pg_type: &PgType) -> DataType {
    match *pg_type {
        PgType::BOOL => DataType::Boolean,
        PgType::INT2 | PgType::INT4 => DataType::Int32,
        PgType::INT8 | PgType::OID => DataType::Int64,
        PgType::FLOAT4 | PgType::FLOAT8 | PgType::NUMERIC => DataType::Float64,
        _ => DataType::Utf8, // everything else → string (safe default per ADR-010)
    }
 }
 fn pg_type_to_string(pg_type: &PgType) -> String {
    format!("{}", pg_type)
 }
 /// Convert a column of Postgres rows to an Arrow array.
 fn rows_to_arrow_column(
    rows: &[tokio_postgres::Row],
    col_idx: usize,
    pg_type: &PgType,
 ) -> Result<ArrayRef, String> {
    match *pg_type {
        PgType::BOOL => {
            let vals: Vec<Option<bool>> = rows.iter()
                .map(|r| r.try_get(col_idx).ok())
                .collect();
            Ok(Arc::new(BooleanArray::from(vals)))
        }
        PgType::INT2 => {
            let vals: Vec<Option<i32>> = rows.iter()
                .map(|r| r.try_get::<_, i16>(col_idx).ok().map(|v| v as i32))
                .collect();
            Ok(Arc::new(Int32Array::from(vals)))
        }
        PgType::INT4 => {
            let vals: Vec<Option<i32>> = rows.iter()
                .map(|r| r.try_get(col_idx).ok())
                .collect();
            Ok(Arc::new(Int32Array::from(vals)))
        }
        PgType::INT8 | PgType::OID => {
            let vals: Vec<Option<i64>> = rows.iter()
                .map(|r| r.try_get(col_idx).ok())
                .collect();
            Ok(Arc::new(Int64Array::from(vals)))
        }
        PgType::FLOAT4 => {
            let vals: Vec<Option<f64>> = rows.iter()
                .map(|r| r.try_get::<_, f32>(col_idx).ok().map(|v| v as f64))
                .collect();
            Ok(Arc::new(Float64Array::from(vals)))
        }
        PgType::FLOAT8 => {
            let vals: Vec<Option<f64>> = rows.iter()
                .map(|r| r.try_get(col_idx).ok())
                .collect();
            Ok(Arc::new(Float64Array::from(vals)))
        }
        _ => {
            // Default: try to get as string
            let vals: Vec<Option<String>> = rows.iter()
                .map(|r| {
                    r.try_get::<_, String>(col_idx).ok()
                        .or_else(|| r.try_get::<_, serde_json::Value>(col_idx).ok().map(|v| v.to_string()))
                        .or_else(|| Some("".to_string()))
                })
                .collect();
            Ok(Arc::new(StringArray::from(vals)))
        }
    }
 }
@@ -1,6 +1,10 @@
 pub mod chatgpt_parser;
 pub mod db_ingest;
 pub mod detect;
 pub mod csv_ingest;
 pub mod json_ingest;
 pub mod pdf_ingest;
 pub mod pipeline;
 pub mod schema_evolution;
 pub mod service;
 pub mod watcher;
@@ -43,7 +43,13 @@ pub async fn ingest_file(
    let (schema, batches) = match file_type {
        FileType::Csv => csv_ingest::parse_csv(content)?,
        FileType::Json | FileType::NdJson => json_ingest::parse_json(content)?,
-        FileType::Pdf => pdf_ingest::parse_pdf(content, filename)?,
+        FileType::Pdf => {
            // Try ChatGPT parser first, fall back to generic PDF
            match crate::chatgpt_parser::parse_chatgpt_pdf(content, filename) {
                Ok(result) => result,
                Err(_) => pdf_ingest::parse_pdf(content, filename)?,
            }
        }
        FileType::Text => parse_text_file(content, filename)?,
        FileType::Unknown => return Err(format!("unknown file type for '{filename}'")),
    };
@@ -94,7 +100,7 @@ pub async fn ingest_file(
    ops::put(store, &storage_key, parquet_bytes).await?;
    tracing::info!("stored {} as {} ({} bytes)", filename, storage_key, parquet_size);
-    // 7. Register in catalog
+    // 7. Register in catalog with rich metadata
    let schema_fp = fingerprint_schema(&schema);
    let now = chrono::Utc::now();
    let obj_ref = ObjectRef {
@@ -104,13 +110,49 @@ pub async fn ingest_file(
        created_at: now,
    };
-    // Use content hash as fingerprint for dedup tracking
+    // Register base dataset
    registry.register(
        safe_name.clone(),
        SchemaFingerprint(hash.clone()),
        vec![obj_ref],
    ).await?;
    // Auto-populate metadata: PII detection, column info, lineage, row count
    let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
    let dataset_sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
    let column_meta: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
        let sens = shared::pii::detect_sensitivity(f.name());
        shared::types::ColumnMeta {
            name: f.name().clone(),
            data_type: f.data_type().to_string(),
            sensitivity: sens.clone(),
            description: String::new(),
            is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
        }
    }).collect();
    let lineage = shared::types::Lineage {
        source_system: format!("{:?}", file_type).to_lowercase(),
        source_file: filename.to_string(),
        ingest_job: format!("ingest-{}", now.timestamp_millis()),
        ingest_timestamp: now,
        parent_datasets: vec![],
    };
    let pii_count = column_meta.iter().filter(|c| c.is_pii).count();
    if pii_count > 0 {
        tracing::info!("auto-detected {} PII columns in '{}'", pii_count, safe_name);
    }
    let _ = registry.update_metadata(&safe_name, catalogd::registry::MetadataUpdate {
        sensitivity: dataset_sensitivity,
        columns: Some(column_meta),
        lineage: Some(lineage),
        row_count: Some(total_rows as u64),
        ..Default::default()
    }).await;
    Ok(IngestResult {
        dataset_name: safe_name,
        file_type: format!("{:?}", file_type),
@@ -0,0 +1,279 @@
 /// Schema evolution: detect changes between ingests and suggest migration rules.
 /// When a source changes format (columns renamed, added, removed, type changed),
 /// the system detects the diff and can auto-map using AI or heuristic matching.
 use arrow::datatypes::{DataType, Schema, SchemaRef};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Arc;
 /// A detected change between two schema versions.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SchemaChange {
    pub change_type: ChangeType,
    pub column: String,
    pub detail: String,
 }
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "snake_case")]
 pub enum ChangeType {
    Added,         // new column in incoming schema
    Removed,       // column in old schema not in new
    TypeChanged,   // same name, different type
    Renamed,       // likely renamed (fuzzy match)
 }
 /// A migration rule — how to map old schema to new.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MigrationRule {
    pub source_column: String,
    pub target_column: String,
    pub transform: String,         // "direct", "cast", "concat", "split", "drop"
    pub confidence: f32,           // 0.0 - 1.0
    pub auto_generated: bool,
 }
 /// Compare two schemas and detect changes.
 pub fn diff_schemas(old: &SchemaRef, new: &SchemaRef) -> Vec<SchemaChange> {
    let mut changes = Vec::new();
    let old_fields: HashMap<&str, &DataType> = old.fields().iter()
        .map(|f| (f.name().as_str(), f.data_type()))
        .collect();
    let new_fields: HashMap<&str, &DataType> = new.fields().iter()
        .map(|f| (f.name().as_str(), f.data_type()))
        .collect();
    // Check for removed columns
    for (name, _dt) in &old_fields {
        if !new_fields.contains_key(name) {
            // Check if it was renamed (fuzzy match)
            let possible_rename = find_similar_column(name, &new_fields, &old_fields);
            if let Some((new_name, confidence)) = possible_rename {
                changes.push(SchemaChange {
                    change_type: ChangeType::Renamed,
                    column: name.to_string(),
                    detail: format!("likely renamed to '{}' (confidence: {:.0}%)", new_name, confidence * 100.0),
                });
            } else {
                changes.push(SchemaChange {
                    change_type: ChangeType::Removed,
                    column: name.to_string(),
                    detail: "column no longer present in incoming data".into(),
                });
            }
        }
    }
    // Check for added or type-changed columns
    for (name, new_dt) in &new_fields {
        match old_fields.get(name) {
            None => {
                // Skip if we already flagged this as a rename target
                let is_rename_target = changes.iter().any(|c| {
                    c.change_type == ChangeType::Renamed && c.detail.contains(name)
                });
                if !is_rename_target {
                    changes.push(SchemaChange {
                        change_type: ChangeType::Added,
                        column: name.to_string(),
                        detail: format!("new column (type: {})", new_dt),
                    });
                }
            }
            Some(old_dt) => {
                if old_dt != new_dt {
                    changes.push(SchemaChange {
                        change_type: ChangeType::TypeChanged,
                        column: name.to_string(),
                        detail: format!("{} → {}", old_dt, new_dt),
                    });
                }
            }
        }
    }
    changes
 }
 /// Generate migration rules from detected changes.
 pub fn generate_rules(changes: &[SchemaChange]) -> Vec<MigrationRule> {
    changes.iter().filter_map(|change| {
        match change.change_type {
            ChangeType::Renamed => {
                // Extract the new name from the detail string
                let new_name = change.detail
                    .split('\'')
                    .nth(1)
                    .unwrap_or(&change.column);
                Some(MigrationRule {
                    source_column: change.column.clone(),
                    target_column: new_name.to_string(),
                    transform: "direct".into(),
                    confidence: 0.8,
                    auto_generated: true,
                })
            }
            ChangeType::TypeChanged => {
                Some(MigrationRule {
                    source_column: change.column.clone(),
                    target_column: change.column.clone(),
                    transform: "cast".into(),
                    confidence: 0.9,
                    auto_generated: true,
                })
            }
            ChangeType::Added => {
                Some(MigrationRule {
                    source_column: String::new(),
                    target_column: change.column.clone(),
                    transform: "new_column".into(),
                    confidence: 1.0,
                    auto_generated: true,
                })
            }
            ChangeType::Removed => {
                Some(MigrationRule {
                    source_column: change.column.clone(),
                    target_column: String::new(),
                    transform: "drop".into(),
                    confidence: 0.7,
                    auto_generated: true,
                })
            }
        }
    }).collect()
 }
 /// Build an AI prompt to suggest migration rules for complex cases.
 pub fn build_migration_prompt(
    old_schema: &SchemaRef,
    new_schema: &SchemaRef,
    changes: &[SchemaChange],
 ) -> String {
    let old_cols: Vec<String> = old_schema.fields().iter()
        .map(|f| format!("  {} ({})", f.name(), f.data_type()))
        .collect();
    let new_cols: Vec<String> = new_schema.fields().iter()
        .map(|f| format!("  {} ({})", f.name(), f.data_type()))
        .collect();
    let change_list: Vec<String> = changes.iter()
        .map(|c| format!("  [{:?}] {} — {}", c.change_type, c.column, c.detail))
        .collect();
    format!(
        "You are a data migration expert. A data source has changed its schema.\n\n\
        OLD SCHEMA:\n{}\n\n\
        NEW SCHEMA:\n{}\n\n\
        DETECTED CHANGES:\n{}\n\n\
        For each change, suggest a migration rule as JSON:\n\
        {{\"source_column\": \"old_name\", \"target_column\": \"new_name\", \"transform\": \"direct|cast|concat|split|drop\", \"confidence\": 0.0-1.0}}\n\n\
        Output ONLY a JSON array of rules, nothing else.",
        old_cols.join("\n"),
        new_cols.join("\n"),
        change_list.join("\n"),
    )
 }
 /// Fuzzy match: find a column in new schema that's similar to a removed old column.
 fn find_similar_column<'a>(
    old_name: &str,
    new_fields: &HashMap<&'a str, &DataType>,
    old_fields: &HashMap<&str, &DataType>,
 ) -> Option<(String, f32)> {
    let old_lower = old_name.to_lowercase();
    let old_parts: Vec<&str> = old_lower.split('_').collect();
    let mut best_match: Option<(String, f32)> = None;
    for (new_name, _) in new_fields {
        // Skip columns that already exist in old schema (not renamed)
        if old_fields.contains_key(new_name) { continue; }
        let new_lower = new_name.to_lowercase();
        let new_parts: Vec<&str> = new_lower.split('_').collect();
        // Exact substring match
        if old_lower.contains(&new_lower) || new_lower.contains(&old_lower) {
            let confidence = 0.85;
            if best_match.as_ref().map_or(true, |(_, c)| confidence > *c) {
                best_match = Some((new_name.to_string(), confidence));
            }
            continue;
        }
        // Shared word parts (e.g., "first_name" → "fname", "full_name")
        let shared: usize = old_parts.iter()
            .filter(|p| new_parts.iter().any(|np| np.contains(*p) || p.contains(np)))
            .count();
        if shared > 0 {
            let confidence = shared as f32 / old_parts.len().max(new_parts.len()) as f32 * 0.75;
            if confidence >= 0.3 && best_match.as_ref().map_or(true, |(_, c)| confidence > *c) {
                best_match = Some((new_name.to_string(), confidence));
            }
        }
    }
    best_match
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    use arrow::datatypes::Field;
    fn schema(fields: Vec<(&str, DataType)>) -> SchemaRef {
        Arc::new(Schema::new(
            fields.into_iter().map(|(n, t)| Field::new(n, t, true)).collect::<Vec<_>>()
        ))
    }
    #[test]
    fn detect_added_column() {
        let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
        let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("email", DataType::Utf8)]);
        let changes = diff_schemas(&old, &new);
        assert_eq!(changes.len(), 1);
        assert_eq!(changes[0].change_type, ChangeType::Added);
        assert_eq!(changes[0].column, "email");
    }
    #[test]
    fn detect_removed_column() {
        let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("age", DataType::Int64)]);
        let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
        let changes = diff_schemas(&old, &new);
        assert!(changes.iter().any(|c| c.column == "age" && c.change_type == ChangeType::Removed));
    }
    #[test]
    fn detect_type_change() {
        let old = schema(vec![("id", DataType::Int64), ("score", DataType::Int64)]);
        let new = schema(vec![("id", DataType::Int64), ("score", DataType::Float64)]);
        let changes = diff_schemas(&old, &new);
        assert_eq!(changes.len(), 1);
        assert_eq!(changes[0].change_type, ChangeType::TypeChanged);
    }
    #[test]
    fn detect_rename() {
        let old = schema(vec![("first_name", DataType::Utf8), ("last_name", DataType::Utf8)]);
        let new = schema(vec![("full_name", DataType::Utf8)]);
        let changes = diff_schemas(&old, &new);
        // Should detect first_name → full_name as a potential rename
        assert!(changes.iter().any(|c| c.change_type == ChangeType::Renamed));
    }
    #[test]
    fn generate_rules_from_changes() {
        let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
        let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("email", DataType::Utf8)]);
        let changes = diff_schemas(&old, &new);
        let rules = generate_rules(&changes);
        assert_eq!(rules.len(), 1);
        assert_eq!(rules[0].transform, "new_column");
    }
 }
@@ -5,12 +5,16 @@ use axum::{
    response::IntoResponse,
    routing::{get, post},
 };
 use bytes::Bytes;
 use object_store::ObjectStore;
 use serde::Deserialize;
 use std::sync::Arc;
 use catalogd::registry::Registry;
-use crate::pipeline;
+use crate::{db_ingest, pipeline};
 use shared::arrow_helpers::record_batch_to_parquet;
 use shared::types::{ObjectRef, SchemaFingerprint};
 use storaged::ops;
 #[derive(Clone)]
 pub struct IngestState {
@@ -22,6 +26,8 @@ pub fn router(state: IngestState) -> Router {
    Router::new()
        .route("/health", get(health))
        .route("/file", post(ingest_file))
        .route("/postgres/tables", post(list_pg_tables))
        .route("/postgres/import", post(import_pg_table))
        .with_state(state)
 }
@@ -31,17 +37,14 @@ async fn health() -> &'static str {
 #[derive(Deserialize)]
 struct IngestQuery {
    /// Override dataset name (otherwise derived from filename)
    name: Option<String>,
 }
 /// Upload a file for ingestion. Accepts multipart/form-data with a "file" field.
 async fn ingest_file(
    State(state): State<IngestState>,
    Query(query): Query<IngestQuery>,
    mut multipart: Multipart,
 ) -> impl IntoResponse {
    // Read the first file field
    let field = match multipart.next_field().await {
        Ok(Some(f)) => f,
        Ok(None) => return Err((StatusCode::BAD_REQUEST, "no file uploaded".to_string())),
@@ -67,3 +70,121 @@ async fn ingest_file(
        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
    }
 }
 // --- PostgreSQL Import ---
 /// List tables in a PostgreSQL database.
 async fn list_pg_tables(
    Json(config): Json<db_ingest::DbConfig>,
 ) -> impl IntoResponse {
    match db_ingest::list_postgres_tables(&config).await {
        Ok(tables) => Ok(Json(tables)),
        Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
    }
 }
 #[derive(Deserialize)]
 struct PgImportRequest {
    #[serde(flatten)]
    config: db_ingest::DbConfig,
    table: String,
    /// Override dataset name (defaults to table name)
    dataset_name: Option<String>,
    /// Max rows to import (None = all)
    limit: Option<usize>,
 }
 /// Import a PostgreSQL table into the lakehouse.
 async fn import_pg_table(
    State(state): State<IngestState>,
    Json(req): Json<PgImportRequest>,
 ) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
    tracing::info!("importing postgres table '{}' from {}:{}/{}",
        req.table, req.config.host, req.config.port, req.config.database);
    // Import from Postgres
    let (schema, batches, db_result) = db_ingest::import_postgres_table(
        &req.config, &req.table, req.limit,
    ).await.map_err(|e| (StatusCode::BAD_GATEWAY, e))?;
    if batches.is_empty() || db_result.rows == 0 {
        return Ok((StatusCode::OK, Json(serde_json::json!({
            "table": req.table,
            "rows": 0,
            "message": "table is empty",
        }))));
    }
    // Convert to Parquet
    let mut all_parquet = Vec::new();
    for batch in &batches {
        let pq = record_batch_to_parquet(batch)
            .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
        all_parquet.extend_from_slice(&pq);
    }
    let dataset_name = req.dataset_name.unwrap_or_else(|| req.table.clone());
    let storage_key = format!("datasets/{}.parquet", dataset_name);
    let parquet_size = all_parquet.len() as u64;
    // Store
    ops::put(&state.store, &storage_key, Bytes::from(all_parquet))
        .await
        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
    // Register
    let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
    let now = chrono::Utc::now();
    state.registry.register(
        dataset_name.clone(),
        SchemaFingerprint(schema_fp.0),
        vec![ObjectRef {
            bucket: "data".to_string(),
            key: storage_key.clone(),
            size_bytes: parquet_size,
            created_at: now,
        }],
    ).await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
    // Auto-populate metadata (PII detection, lineage)
    let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
    let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
    let columns: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
        let sens = shared::pii::detect_sensitivity(f.name());
        shared::types::ColumnMeta {
            name: f.name().clone(),
            data_type: f.data_type().to_string(),
            sensitivity: sens.clone(),
            description: String::new(),
            is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
        }
    }).collect();
    let lineage = shared::types::Lineage {
        source_system: "postgresql".to_string(),
        source_file: format!("{}:{}/{}.{}", req.config.host, req.config.port, req.config.database, req.table),
        ingest_job: format!("pg-import-{}", now.timestamp_millis()),
        ingest_timestamp: now,
        parent_datasets: vec![],
    };
    let _ = state.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
        sensitivity,
        columns: Some(columns),
        lineage: Some(lineage),
        row_count: Some(db_result.rows as u64),
        ..Default::default()
    }).await;
    tracing::info!("imported '{}' from postgres: {} rows → {}", dataset_name, db_result.rows, storage_key);
    Ok((StatusCode::CREATED, Json(serde_json::json!({
        "dataset_name": dataset_name,
        "table": req.table,
        "rows": db_result.rows,
        "columns": db_result.columns,
        "schema": db_result.schema_detected,
        "storage_key": storage_key,
        "size_bytes": parquet_size,
    }))))
 }
@@ -0,0 +1,122 @@
 /// File watcher: monitors a directory for new files and auto-ingests them.
 /// Drop a CSV, JSON, PDF, or text file → auto-detected → Parquet → queryable.
 /// Processed files are moved to a "processed/" subdirectory.
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::time;
 use catalogd::registry::Registry;
 use object_store::ObjectStore;
 use crate::pipeline;
 /// Watch config.
 pub struct WatchConfig {
    pub watch_dir: PathBuf,
    pub processed_dir: PathBuf,
    pub failed_dir: PathBuf,
    pub poll_interval: Duration,
 }
 impl Default for WatchConfig {
    fn default() -> Self {
        Self {
            watch_dir: PathBuf::from("./inbox"),
            processed_dir: PathBuf::from("./inbox/processed"),
            failed_dir: PathBuf::from("./inbox/failed"),
            poll_interval: Duration::from_secs(10),
        }
    }
 }
 /// Start the file watcher as a background task.
 pub fn spawn_watcher(
    config: WatchConfig,
    store: Arc<dyn ObjectStore>,
    registry: Registry,
 ) -> tokio::task::JoinHandle<()> {
    // Create directories
    std::fs::create_dir_all(&config.watch_dir).ok();
    std::fs::create_dir_all(&config.processed_dir).ok();
    std::fs::create_dir_all(&config.failed_dir).ok();
    tracing::info!("file watcher started: watching {}", config.watch_dir.display());
    tokio::spawn(async move {
        let mut interval = time::interval(config.poll_interval);
        loop {
            interval.tick().await;
            if let Err(e) = process_inbox(&config, &store, &registry).await {
                tracing::error!("watcher error: {e}");
            }
        }
    })
 }
 /// Scan inbox directory and process any new files.
 async fn process_inbox(
    config: &WatchConfig,
    store: &Arc<dyn ObjectStore>,
    registry: &Registry,
 ) -> Result<(), String> {
    let entries = std::fs::read_dir(&config.watch_dir)
        .map_err(|e| format!("can't read inbox: {e}"))?;
    for entry in entries {
        let entry = match entry {
            Ok(e) => e,
            Err(_) => continue,
        };
        let path = entry.path();
        // Skip directories and hidden files
        if path.is_dir() || path.file_name().map_or(true, |n| n.to_string_lossy().starts_with('.')) {
            continue;
        }
        let filename = path.file_name().unwrap().to_string_lossy().to_string();
        tracing::info!("watcher: found new file '{}'", filename);
        // Read file
        let content = match std::fs::read(&path) {
            Ok(c) => c,
            Err(e) => {
                tracing::error!("watcher: can't read '{}': {e}", filename);
                move_file(&path, &config.failed_dir.join(&filename));
                continue;
            }
        };
        // Ingest
        match pipeline::ingest_file(&filename, &content, None, store, registry).await {
            Ok(result) => {
                if result.deduplicated {
                    tracing::info!("watcher: '{}' already ingested, skipping", filename);
                } else {
                    tracing::info!("watcher: ingested '{}' → {} ({} rows, {} cols)",
                        filename, result.dataset_name, result.rows, result.columns);
                }
                // Move to processed
                move_file(&path, &config.processed_dir.join(&filename));
            }
            Err(e) => {
                tracing::error!("watcher: failed to ingest '{}': {e}", filename);
                move_file(&path, &config.failed_dir.join(&filename));
            }
        }
    }
    Ok(())
 }
 fn move_file(from: &Path, to: &Path) {
    if let Err(e) = std::fs::rename(from, to) {
        // rename fails across filesystems, try copy+delete
        if let Err(e2) = std::fs::copy(from, to).and_then(|_| std::fs::remove_file(from)) {
            tracing::warn!("can't move {} → {}: {e}, copy failed: {e2}", from.display(), to.display());
        }
    }
 }
@@ -1,6 +1,7 @@
 pub mod cache;
 pub mod context;
 pub mod delta;
 pub mod paged;
 pub mod service;
 pub mod workspace;
 pub mod workspace_service;
@@ -0,0 +1,222 @@
 /// Paged query results with server-side cursor.
 /// Large results are stored server-side, client fetches pages on demand.
 /// Pattern matches the embedding supervisor: chunk → process → retry.
 use arrow::array::RecordBatch;
 use arrow::json::writer::{JsonArray, Writer as JsonWriter};
 use serde::Serialize;
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 use crate::context::QueryEngine;
 /// A stored query result that clients page through.
 #[derive(Clone)]
 pub struct StoredResult {
    pub query_id: String,
    pub sql: String,
    pub columns: Vec<ColumnInfo>,
    pub total_rows: usize,
    pub batches: Vec<RecordBatch>,
    pub created_at: std::time::Instant,
 }
 #[derive(Clone, Serialize)]
 pub struct ColumnInfo {
    pub name: String,
    pub data_type: String,
 }
 /// Response for initial query — metadata only, no rows.
 #[derive(Serialize)]
 pub struct QueryHandle {
    pub query_id: String,
    pub total_rows: usize,
    pub columns: Vec<ColumnInfo>,
    pub total_pages: usize,
    pub page_size: usize,
 }
 /// A page of results.
 #[derive(Serialize)]
 pub struct PageResult {
    pub query_id: String,
    pub page: usize,
    pub page_size: usize,
    pub total_rows: usize,
    pub total_pages: usize,
    pub rows: serde_json::Value,
    pub row_count: usize,
 }
 /// Server-side result store with TTL.
 #[derive(Clone)]
 pub struct ResultStore {
    results: Arc<RwLock<HashMap<String, StoredResult>>>,
    default_page_size: usize,
    max_results: usize,
 }
 impl ResultStore {
    pub fn new(default_page_size: usize, max_results: usize) -> Self {
        Self {
            results: Arc::new(RwLock::new(HashMap::new())),
            default_page_size,
            max_results,
        }
    }
    /// Execute query and store results. Returns handle with metadata.
    pub async fn execute_and_store(
        &self,
        engine: &QueryEngine,
        sql: &str,
    ) -> Result<QueryHandle, String> {
        let batches = engine.query(sql).await?;
        if batches.is_empty() {
            let qid = format!("qr-{}", chrono::Utc::now().timestamp_millis());
            return Ok(QueryHandle {
                query_id: qid,
                total_rows: 0,
                columns: vec![],
                total_pages: 0,
                page_size: self.default_page_size,
            });
        }
        let schema = batches[0].schema();
        let columns: Vec<ColumnInfo> = schema.fields().iter().map(|f| ColumnInfo {
            name: f.name().clone(),
            data_type: f.data_type().to_string(),
        }).collect();
        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
        let total_pages = (total_rows + self.default_page_size - 1) / self.default_page_size;
        let qid = format!("qr-{}", chrono::Utc::now().timestamp_millis());
        let result = StoredResult {
            query_id: qid.clone(),
            sql: sql.to_string(),
            columns: columns.clone(),
            total_rows,
            batches,
            created_at: std::time::Instant::now(),
        };
        // Store (evict old results if at capacity)
        let mut store = self.results.write().await;
        if store.len() >= self.max_results {
            // Evict oldest
            if let Some(oldest_key) = store.iter()
                .min_by_key(|(_, v)| v.created_at)
                .map(|(k, _)| k.clone())
            {
                store.remove(&oldest_key);
            }
        }
        store.insert(qid.clone(), result);
        Ok(QueryHandle {
            query_id: qid,
            total_rows,
            columns,
            total_pages,
            page_size: self.default_page_size,
        })
    }
    /// Get a page of results. Handles chunking and retry-safe.
    pub async fn get_page(
        &self,
        query_id: &str,
        page: usize,
        page_size: Option<usize>,
    ) -> Result<PageResult, String> {
        let store = self.results.read().await;
        let result = store.get(query_id)
            .ok_or_else(|| format!("query result not found: {query_id} (may have expired)"))?;
        let ps = page_size.unwrap_or(self.default_page_size);
        let start_row = page * ps;
        if start_row >= result.total_rows {
            return Ok(PageResult {
                query_id: query_id.to_string(),
                page,
                page_size: ps,
                total_rows: result.total_rows,
                total_pages: (result.total_rows + ps - 1) / ps,
                rows: serde_json::Value::Array(vec![]),
                row_count: 0,
            });
        }
        let end_row = (start_row + ps).min(result.total_rows);
        // Extract the right rows from batches
        let page_batches = slice_batches(&result.batches, start_row, end_row);
        let rows = batches_to_json(&page_batches)?;
        let row_count = rows.as_array().map(|a| a.len()).unwrap_or(0);
        Ok(PageResult {
            query_id: query_id.to_string(),
            page,
            page_size: ps,
            total_rows: result.total_rows,
            total_pages: (result.total_rows + ps - 1) / ps,
            rows,
            row_count,
        })
    }
    pub fn default_page_size(&self) -> usize {
        self.default_page_size
    }
 }
 /// Slice record batches to extract rows [start, end).
 fn slice_batches(batches: &[RecordBatch], start: usize, end: usize) -> Vec<RecordBatch> {
    let mut result = Vec::new();
    let mut current_offset = 0;
    for batch in batches {
        let batch_end = current_offset + batch.num_rows();
        if batch_end <= start {
            current_offset = batch_end;
            continue;
        }
        if current_offset >= end {
            break;
        }
        let local_start = if start > current_offset { start - current_offset } else { 0 };
        let local_end = if end < batch_end { end - current_offset } else { batch.num_rows() };
        let length = local_end - local_start;
        if length > 0 {
            result.push(batch.slice(local_start, length));
        }
        current_offset = batch_end;
    }
    result
 }
 fn batches_to_json(batches: &[RecordBatch]) -> Result<serde_json::Value, String> {
    if batches.is_empty() {
        return Ok(serde_json::Value::Array(vec![]));
    }
    let mut buf = Vec::new();
    let mut writer = JsonWriter::<_, JsonArray>::new(&mut buf);
    for batch in batches {
        writer.write(batch).map_err(|e| format!("JSON write: {e}"))?;
    }
    writer.finish().map_err(|e| format!("JSON finish: {e}"))?;
    drop(writer);
    serde_json::from_slice(&buf).map_err(|e| format!("JSON parse: {e}"))
 }
@@ -2,7 +2,7 @@ use arrow::array::RecordBatch;
 use arrow::json::writer::{JsonArray, Writer as JsonWriter};
 use axum::{
    Json, Router,
-    extract::State,
+    extract::{Path, Query, State},
    http::StatusCode,
    response::IntoResponse,
    routing::{get, post},
@@ -12,16 +12,29 @@ use serde::{Deserialize, Serialize};
 use crate::cache::CacheStats;
 use crate::context::QueryEngine;
 use crate::delta;
 use crate::paged::ResultStore;
 #[derive(Clone)]
 pub struct QueryState {
    pub engine: QueryEngine,
    pub result_store: ResultStore,
 }
 pub fn router(engine: QueryEngine) -> Router {
    let state = QueryState {
        engine: engine.clone(),
        result_store: ResultStore::new(100, 50), // 100 rows/page, keep 50 results
    };
    Router::new()
        .route("/health", get(health))
        .route("/sql", post(execute_query))
        .route("/paged", post(paged_query))
        .route("/page/{query_id}/{page}", get(get_page))
        .route("/cache/pin", post(pin_dataset))
        .route("/cache/evict", post(evict_dataset))
        .route("/cache/stats", get(cache_stats))
        .route("/compact", post(compact_dataset))
-        .with_state(engine)
+        .with_state(state)
 }
 async fn health() -> &'static str {
@@ -60,12 +73,12 @@ fn batches_to_json(batches: &[RecordBatch]) -> Result<serde_json::Value, String>
 }
 async fn execute_query(
-    State(engine): State<QueryEngine>,
+    State(state): State<QueryState>,
    Json(req): Json<QueryRequest>,
 ) -> impl IntoResponse {
    tracing::info!("executing query: {}", req.sql);
-    match engine.query(&req.sql).await {
+    match state.engine.query(&req.sql).await {
        Ok(batches) => {
            if batches.is_empty() {
                return Ok(Json(QueryResponse {
@@ -96,6 +109,35 @@ async fn execute_query(
    }
 }
 // --- Paged Queries (large result sets) ---
 async fn paged_query(
    State(state): State<QueryState>,
    Json(req): Json<QueryRequest>,
 ) -> impl IntoResponse {
    tracing::info!("paged query: {}", req.sql);
    match state.result_store.execute_and_store(&state.engine, &req.sql).await {
        Ok(handle) => Ok(Json(handle)),
        Err(e) => Err((StatusCode::BAD_REQUEST, e)),
    }
 }
 #[derive(Deserialize)]
 struct PageQuery {
    size: Option<usize>,
 }
 async fn get_page(
    State(state): State<QueryState>,
    Path((query_id, page)): Path<(String, usize)>,
    Query(q): Query<PageQuery>,
 ) -> impl IntoResponse {
    match state.result_store.get_page(&query_id, page, q.size).await {
        Ok(result) => Ok(Json(result)),
        Err(e) => Err((StatusCode::NOT_FOUND, e)),
    }
 }
 // --- Cache Management ---
 #[derive(Deserialize)]
@@ -104,28 +146,28 @@ struct CacheRequest {
 }
 async fn pin_dataset(
-    State(engine): State<QueryEngine>,
+    State(state): State<QueryState>,
    Json(req): Json<CacheRequest>,
 ) -> impl IntoResponse {
-    match engine.pin_dataset(&req.dataset).await {
+    match state.engine.pin_dataset(&req.dataset).await {
        Ok(()) => Ok((StatusCode::OK, format!("pinned: {}", req.dataset))),
        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
    }
 }
 async fn evict_dataset(
-    State(engine): State<QueryEngine>,
+    State(state): State<QueryState>,
    Json(req): Json<CacheRequest>,
 ) -> impl IntoResponse {
-    if engine.cache().evict(&req.dataset).await {
+    if state.engine.cache().evict(&req.dataset).await {
        (StatusCode::OK, format!("evicted: {}", req.dataset))
    } else {
        (StatusCode::NOT_FOUND, format!("not cached: {}", req.dataset))
    }
 }
-async fn cache_stats(State(engine): State<QueryEngine>) -> impl IntoResponse {
+async fn cache_stats(State(state): State<QueryState>) -> impl IntoResponse {
-    let stats = engine.cache().stats().await;
+    let stats = state.engine.cache().stats().await;
    Json(stats)
 }
@@ -139,11 +181,11 @@ struct CompactRequest {
 }
 async fn compact_dataset(
-    State(engine): State<QueryEngine>,
+    State(state): State<QueryState>,
    Json(req): Json<CompactRequest>,
 ) -> impl IntoResponse {
    match delta::compact(
-        engine.store(),
+        state.engine.store(),
        &req.dataset,
        &req.base_key,
        req.primary_key.as_deref(),
@@ -2,3 +2,4 @@ pub mod types;
 pub mod errors;
 pub mod arrow_helpers;
 pub mod config;
 pub mod pii;
@@ -0,0 +1,116 @@
 /// Auto-detect PII columns by name patterns.
 /// Conservative: flags likely PII, doesn't miss obvious cases.
 use crate::types::Sensitivity;
 /// Check if a column name suggests PII content.
 pub fn detect_sensitivity(column_name: &str) -> Option<Sensitivity> {
    let lower = column_name.to_lowercase();
    // Direct PII identifiers
    if matches!(lower.as_str(),
        "ssn" | "social_security" | "social_security_number" |
        "sin" | "national_id" | "passport" | "passport_number" |
        "drivers_license" | "driver_license" | "dl_number"
    ) {
        return Some(Sensitivity::Pii);
    }
    // Names
    if lower.contains("first_name") || lower.contains("last_name") ||
       lower.contains("full_name") || lower.contains("middle_name") ||
       lower == "name" || lower == "fname" || lower == "lname" {
        return Some(Sensitivity::Pii);
    }
    // Contact info
    if lower.contains("email") || lower.contains("e_mail") ||
       lower.contains("phone") || lower.contains("mobile") || lower.contains("cell") ||
       lower.contains("fax") || lower == "tel" || lower == "telephone" {
        return Some(Sensitivity::Pii);
    }
    // Address
    if lower.contains("address") || lower.contains("street") ||
       (lower.contains("zip") && !lower.contains("unzip")) ||
       lower == "postal_code" || lower == "postcode" {
        return Some(Sensitivity::Pii);
    }
    // Financial
    if lower.contains("salary") || lower.contains("wage") ||
       lower.contains("pay_rate") || lower.contains("bill_rate") ||
       lower.contains("compensation") || lower.contains("revenue") ||
       lower.contains("bank_account") || lower.contains("routing_number") ||
       lower.contains("credit_card") {
        return Some(Sensitivity::Financial);
    }
    // Health
    if lower.contains("diagnosis") || lower.contains("medication") ||
       lower.contains("medical") || lower.contains("health") ||
       lower.contains("patient_id") || lower.contains("mrn") {
        return Some(Sensitivity::Phi);
    }
    // Date of birth
    if lower == "dob" || lower == "date_of_birth" || lower == "birthdate" || lower == "birth_date" {
        return Some(Sensitivity::Pii);
    }
    None
 }
 /// Classify all columns and return the highest sensitivity found.
 pub fn detect_dataset_sensitivity(column_names: &[&str]) -> Option<Sensitivity> {
    let mut highest: Option<Sensitivity> = None;
    for name in column_names {
        if let Some(sens) = detect_sensitivity(name) {
            highest = Some(match (&highest, &sens) {
                (None, s) => s.clone(),
                (Some(Sensitivity::Public), s) => s.clone(),
                (Some(Sensitivity::Internal), s) if !matches!(s, Sensitivity::Public | Sensitivity::Internal) => s.clone(),
                (Some(Sensitivity::Financial), Sensitivity::Pii | Sensitivity::Phi) => sens.clone(),
                (Some(Sensitivity::Pii), Sensitivity::Phi) => Sensitivity::Phi,
                (existing, _) => existing.clone().unwrap(),
            });
        }
    }
    highest
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[test]
    fn detects_email_as_pii() {
        assert_eq!(detect_sensitivity("email"), Some(Sensitivity::Pii));
        assert_eq!(detect_sensitivity("contact_email"), Some(Sensitivity::Pii));
    }
    #[test]
    fn detects_salary_as_financial() {
        assert_eq!(detect_sensitivity("salary"), Some(Sensitivity::Financial));
        assert_eq!(detect_sensitivity("bill_rate"), Some(Sensitivity::Financial));
    }
    #[test]
    fn detects_ssn() {
        assert_eq!(detect_sensitivity("ssn"), Some(Sensitivity::Pii));
    }
    #[test]
    fn non_sensitive_returns_none() {
        assert_eq!(detect_sensitivity("status"), None);
        assert_eq!(detect_sensitivity("created_at"), None);
    }
    #[test]
    fn dataset_sensitivity_picks_highest() {
        let cols = vec!["id", "name", "email", "status"];
        assert_eq!(detect_dataset_sensitivity(&cols), Some(Sensitivity::Pii));
    }
 }
@@ -30,7 +30,50 @@ pub struct ObjectRef {
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct SchemaFingerprint(pub String);
-/// Dataset manifest — the catalog's view of a dataset.
+/// Sensitivity classification for a field or dataset.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
 pub enum Sensitivity {
    Public,      // anyone can see
    Internal,    // employees only
    Pii,         // personally identifiable (name, email, phone, SSN)
    Phi,         // protected health info (HIPAA)
    Financial,   // compensation, billing, revenue
 }
 /// Column-level metadata.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ColumnMeta {
    pub name: String,
    pub data_type: String,
    #[serde(default)]
    pub sensitivity: Option<Sensitivity>,
    #[serde(default)]
    pub description: String,
    #[serde(default)]
    pub is_pii: bool,
 }
 /// Data lineage — where this dataset came from.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Lineage {
    pub source_system: String,    // "csv_upload", "mysql_import", "api_ingest"
    pub source_file: String,      // original filename or URI
    pub ingest_job: String,       // job ID that created this dataset
    pub ingest_timestamp: chrono::DateTime<chrono::Utc>,
    #[serde(default)]
    pub parent_datasets: Vec<String>, // derived from these datasets
 }
 /// Freshness contract — how often this data should be updated.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct FreshnessContract {
    pub expected_frequency: String, // "daily", "weekly", "hourly", "manual"
    pub stale_after_hours: u32,     // alert if not updated within this many hours
 }
 /// Dataset manifest — the catalog's complete view of a dataset.
 /// All new fields use #[serde(default)] for backward compatibility with existing manifests.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DatasetManifest {
    pub id: DatasetId,
@@ -39,4 +82,38 @@ pub struct DatasetManifest {
    pub objects: Vec<ObjectRef>,
    pub created_at: chrono::DateTime<chrono::Utc>,
    pub updated_at: chrono::DateTime<chrono::Utc>,
    // --- Rich metadata (Phase 10) ---
    /// Human-readable description (auto-generated on ingest or manual)
    #[serde(default)]
    pub description: String,
    /// Who owns this dataset
    #[serde(default)]
    pub owner: String,
    /// Overall sensitivity classification
    #[serde(default)]
    pub sensitivity: Option<Sensitivity>,
    /// Column-level metadata including sensitivity and descriptions
    #[serde(default)]
    pub columns: Vec<ColumnMeta>,
    /// Where this data came from
    #[serde(default)]
    pub lineage: Option<Lineage>,
    /// How fresh the data should be
    #[serde(default)]
    pub freshness: Option<FreshnessContract>,
    /// Free-form tags for discovery
    #[serde(default)]
    pub tags: Vec<String>,
    /// Row count (updated on ingest/compact)
    #[serde(default)]
    pub row_count: Option<u64>,
 }
@@ -162,3 +162,46 @@ th {
 }
 td { padding: 8px 14px; border-bottom: 1px solid var(--border); }
 tr:hover td { background: var(--accent-glow); }
 /* Dashboard */
 .dashboard-hero { margin-bottom: 28px; }
 .dashboard-hero h2 { font-size: 32px; font-weight: 700; }
 .stat-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); gap: 12px; margin-bottom: 32px; }
 .stat-card {
    background: var(--surface); border: 1px solid var(--border); border-radius: 10px;
    padding: 20px 16px; text-align: center;
 }
 .stat-card.accent { border-color: var(--accent); background: var(--accent-glow); }
 .stat-value { font-size: 28px; font-weight: 700; color: var(--text); }
 .stat-card.accent .stat-value { color: var(--accent); }
 .stat-label { font-size: 11px; color: var(--text-dim); margin-top: 4px; text-transform: uppercase; letter-spacing: 0.05em; }
 .architecture-section, .phases-section { margin-bottom: 28px; }
 .architecture-section h3, .phases-section h3 { font-size: 16px; font-weight: 600; color: var(--text-dim); margin-bottom: 12px; text-transform: uppercase; letter-spacing: 0.08em; }
 .arch-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 10px; }
 .arch-card {
    background: var(--surface); border: 1px solid var(--border); border-radius: 8px;
    padding: 14px; border-left: 3px solid var(--accent);
 }
 .arch-title { font-weight: 600; font-size: 13px; margin-bottom: 4px; color: var(--accent); }
 .arch-items { font-size: 11px; color: var(--text-dim); line-height: 1.6; }
 .phase-list { display: flex; flex-direction: column; gap: 2px; }
 .phase-item {
    display: grid; grid-template-columns: 40px 180px 1fr; gap: 12px; align-items: center;
    padding: 8px 12px; border-radius: 6px; font-size: 12px;
    background: var(--surface); border-left: 3px solid var(--success);
 }
 .phase-num { font-weight: 700; color: var(--accent); text-align: center; }
 .phase-name { font-weight: 600; }
 .phase-detail { color: var(--text-dim); }
 /* Ingest */
 .table-list { margin-top: 12px; }
 .table-item {
    display: flex; justify-content: space-between; align-items: center;
    padding: 8px 12px; border-bottom: 1px solid var(--border); font-size: 13px;
 }
 .table-item:hover { background: var(--accent-glow); }
@@ -0,0 +1,23 @@
 {
  "id": "6d0002ef-28de-4b92-abed-0bfb7ab5fb6f",
  "name": "candidates",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/candidates.parquet",
      "size_bytes": 10592165,
      "created_at": "2026-03-28T01:54:06.898505222Z"
    }
  ],
  "created_at": "2026-03-28T01:54:06.898506121Z",
  "updated_at": "2026-03-28T01:54:06.898506121Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "73a4ef71-5b2a-4124-a2c2-fcefdde99624",
  "name": "job_orders",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/job_orders.parquet",
      "size_bytes": 905534,
      "created_at": "2026-03-28T01:54:07.007332060Z"
    }
  ],
  "created_at": "2026-03-28T01:54:07.007332775Z",
  "updated_at": "2026-03-28T01:54:07.007332775Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "b56694bb-f4f7-447d-b4d6-ac89103a6a3a",
  "name": "call_log",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/call_log.parquet",
      "size_bytes": 35951077,
      "created_at": "2026-03-28T01:54:15.457573021Z"
    }
  ],
  "created_at": "2026-03-28T01:54:15.457573701Z",
  "updated_at": "2026-03-28T01:54:15.457573701Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "daa5a11b-6898-40ba-a811-3bd4b6f5b750",
  "name": "email_log",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/email_log.parquet",
      "size_bytes": 16768671,
      "created_at": "2026-03-28T01:54:17.356056562Z"
    }
  ],
  "created_at": "2026-03-28T01:54:17.356057529Z",
  "updated_at": "2026-03-28T01:54:17.356057529Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "dd396ee4-3a1a-4939-af60-84d409eebed4",
  "name": "clients",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/clients.parquet",
      "size_bytes": 21971,
      "created_at": "2026-03-28T01:54:06.904379300Z"
    }
  ],
  "created_at": "2026-03-28T01:54:06.904379793Z",
  "updated_at": "2026-03-28T01:54:06.904379793Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "f85285e3-111a-4a55-aa58-05fd740ed862",
  "name": "timesheets",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/timesheets.parquet",
      "size_bytes": 17539932,
      "created_at": "2026-03-28T01:54:11.947773535Z"
    }
  ],
  "created_at": "2026-03-28T01:54:11.947778065Z",
  "updated_at": "2026-03-28T01:54:11.947778065Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "ffa72582-dc11-4454-a46e-35839a7d04bb",
  "name": "placements",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/placements.parquet",
      "size_bytes": 1213820,
      "created_at": "2026-03-28T01:54:07.109218264Z"
    }
  ],
  "created_at": "2026-03-28T01:54:07.109219070Z",
  "updated_at": "2026-03-28T01:54:07.109219070Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -109,20 +109,43 @@ async fn fetch_health(path: &str) -> Result<String, String> {
    resp.text().await.map_err(|e| e.to_string())
 }
-/// Get schema context for all datasets (used for AI SQL generation)
+/// Cached schema context — built once, reused across questions.
 static SCHEMA_CACHE: std::sync::OnceLock<std::sync::Mutex<Option<String>>> = std::sync::OnceLock::new();
 async fn get_schema_context_cached(datasets: &[Dataset]) -> String {
    // Check cache first
    {
        let cache = SCHEMA_CACHE.get_or_init(|| std::sync::Mutex::new(None));
        if let Ok(guard) = cache.lock() {
            if let Some(ref cached) = *guard {
                return cached.clone();
            }
        }
    }
    // Build and cache
    let ctx = get_schema_context(datasets).await;
    if let Some(cache) = SCHEMA_CACHE.get() {
        if let Ok(mut guard) = cache.lock() {
            *guard = Some(ctx.clone());
        }
    }
    ctx
 }
 /// Get schema context for datasets (used for AI SQL generation).
 async fn get_schema_context(datasets: &[Dataset]) -> String {
-    let mut ctx = String::from("Available tables:\n\n");
+    let core_tables = ["candidates", "clients", "job_orders", "placements", "timesheets", "call_log", "email_log"];
-    for ds in datasets {
+    let mut ctx = String::from("DATABASE SCHEMA:\n\n");
    for ds in datasets.iter().filter(|d| core_tables.contains(&d.name.as_str())) {
        let desc = run_sql(&format!("DESCRIBE {}", ds.name)).await;
        match desc {
            Ok(resp) => {
-                ctx.push_str(&format!("TABLE: {}\n", ds.name));
+                ctx.push_str(&format!("TABLE: {}\n  Columns:\n", ds.name));
                if let Some(rows) = resp.rows.as_array() {
                    for row in rows {
                        let col = row.get("column_name").and_then(|v| v.as_str()).unwrap_or("?");
                        let dt = row.get("data_type").and_then(|v| v.as_str()).unwrap_or("?");
-                        let nullable = row.get("is_nullable").and_then(|v| v.as_str()).unwrap_or("?");
+                        ctx.push_str(&format!("    {}.{} ({})\n", ds.name, col, dt));
                        ctx.push_str(&format!("  - {} ({}, nullable={})\n", col, dt, nullable));
                    }
                }
                ctx.push('\n');
@@ -132,15 +155,29 @@ async fn get_schema_context(datasets: &[Dataset]) -> String {
            }
        }
    }
    // Add relationship hints so the model knows how to JOIN
    ctx.push_str("RELATIONSHIPS:\n");
    ctx.push_str("  candidates.candidate_id = placements.candidate_id\n");
    ctx.push_str("  candidates.candidate_id = timesheets.candidate_id\n");
    ctx.push_str("  candidates.candidate_id = call_log.candidate_id\n");
    ctx.push_str("  candidates.candidate_id = email_log.candidate_id\n");
    ctx.push_str("  clients.client_id = job_orders.client_id\n");
    ctx.push_str("  clients.client_id = placements.client_id\n");
    ctx.push_str("  clients.client_id = timesheets.client_id\n");
    ctx.push_str("  job_orders.job_order_id = placements.job_order_id\n");
    ctx.push_str("  placements.placement_id = timesheets.placement_id\n");
    ctx.push_str("\nNOTE: 'vertical' is only in candidates, clients, and job_orders. To get vertical for timesheets or placements, JOIN to those tables.\n");
    ctx
 }
 // --- Tabs ---
 #[derive(Clone, PartialEq)]
 enum Tab {
    Dashboard,
    Ask,
    Explore,
    Sql,
    Ingest,
    Status,
 }
@@ -177,6 +214,11 @@ fn App() -> Element {
            div { class: "header",
                h1 { "LAKEHOUSE" }
                div { class: "tabs",
                    button {
                        class: if *active_tab.read() == Tab::Dashboard { "tab active" } else { "tab" },
                        onclick: move |_| active_tab.set(Tab::Dashboard),
                        "Dashboard"
                    }
                    button {
                        class: if *active_tab.read() == Tab::Ask { "tab active" } else { "tab" },
                        onclick: move |_| active_tab.set(Tab::Ask),
@@ -192,6 +234,11 @@ fn App() -> Element {
                        onclick: move |_| active_tab.set(Tab::Sql),
                        "SQL"
                    }
                    button {
                        class: if *active_tab.read() == Tab::Ingest { "tab active" } else { "tab" },
                        onclick: move |_| active_tab.set(Tab::Ingest),
                        "Ingest"
                    }
                    button {
                        class: if *active_tab.read() == Tab::Status { "tab active" } else { "tab" },
                        onclick: move |_| active_tab.set(Tab::Status),
@@ -208,9 +255,11 @@ fn App() -> Element {
            div { class: "content-full",
                match *active_tab.read() {
                    Tab::Dashboard => rsx! { DashboardPanel {} },
                    Tab::Ask => rsx! { AskPanel { datasets: datasets.read().clone() } },
                    Tab::Explore => rsx! { ExplorePanel { datasets: datasets.read().clone() } },
                    Tab::Sql => rsx! { SqlPanel {} },
                    Tab::Ingest => rsx! { IngestPanel {} },
                    Tab::Status => rsx! { StatusPanel {} },
                }
            }
@@ -241,27 +290,56 @@ fn AskPanel(datasets: Vec<Dataset>) -> Element {
            // Step 1: Get schema context
            step.set("reading schemas...".into());
-            let schema_ctx = get_schema_context(&ds).await;
+            let schema_ctx = get_schema_context_cached(&ds).await;
            // Step 2: Generate SQL
            step.set("writing SQL...".into());
            let prompt = format!(
-                "You are a SQL assistant for a data lakehouse using Apache DataFusion (PostgreSQL-compatible SQL).\n\n\
+                "You are a SQL assistant. You write Apache DataFusion SQL (PostgreSQL-compatible).\n\n\
                CRITICAL: You MUST only use column names that appear in the schema below. Do NOT invent or guess column names.\n\n\
                {schema_ctx}\n\
                User question: {q}\n\n\
-                Write a SQL query that answers this question. Output ONLY the SQL query, nothing else. No markdown, no explanation, no backticks."
+                Rules:\n\
                - ONLY use table and column names listed above\n\
                - If unsure about a column name, pick the closest match from the schema\n\
                - Output ONLY the SQL query. No markdown, no explanation, no backticks."
            );
            match ai_generate(&prompt, 512).await {
                Ok(resp) => {
-                    let sql = resp.text.trim().to_string();
+                    let sql = clean_sql(&resp.text);
                    generated_sql.set(Some(sql.clone()));
                    // Step 3: Execute
                    step.set("running query...".into());
                    let query_result = run_sql(&sql).await;
                    // Step 3b: If schema error, retry with the error as feedback
                    if let Err(ref err) = query_result {
                        if err.contains("error") {
                            step.set("fixing SQL...".into());
                            let retry_prompt = format!(
                                "The SQL you wrote had an error:\n{err}\n\n\
                                {schema_ctx}\n\
                                Original question: {q}\n\n\
                                Write a CORRECTED SQL query using ONLY the columns listed in the schema. Output ONLY SQL."
                            );
                            if let Ok(retry_resp) = ai_generate(&retry_prompt, 512).await {
                                let retry_sql = clean_sql(&retry_resp.text);
                                generated_sql.set(Some(retry_sql.clone()));
                                step.set("running corrected query...".into());
                                let retry_result = run_sql(&retry_sql).await;
                                result.set(Some(retry_result));
                            } else {
                                result.set(Some(query_result));
                            }
                        } else {
                            result.set(Some(query_result));
                        }
                    } else {
                        result.set(Some(query_result));
                    }
                }
                Err(e) => {
                    result.set(Some(Err(format!("AI error: {e}"))));
                }
@@ -295,22 +373,45 @@ fn AskPanel(datasets: Vec<Dataset>) -> Element {
                                generated_sql.set(None);
                                result.set(None);
                                step.set("reading schemas...".into());
-                                let schema_ctx = get_schema_context(&ds).await;
+                                let schema_ctx = get_schema_context_cached(&ds).await;
                                step.set("writing SQL...".into());
                                let prompt = format!(
-                                    "You are a SQL assistant for a data lakehouse using Apache DataFusion (PostgreSQL-compatible SQL).\n\n\
+                                    "You are a SQL assistant. You write Apache DataFusion SQL (PostgreSQL-compatible).\n\n\
                                    CRITICAL: You MUST only use column names that appear in the schema below. Do NOT invent or guess column names.\n\n\
                                    {schema_ctx}\n\
                                    User question: {q}\n\n\
-                                    Write a SQL query that answers this question. Output ONLY the SQL query, nothing else. No markdown, no explanation, no backticks."
+                                    Rules:\n\
                                    - ONLY use table and column names listed above\n\
                                    - If unsure about a column name, pick the closest match from the schema\n\
                                    - Output ONLY the SQL query. No markdown, no explanation, no backticks."
                                );
                                match ai_generate(&prompt, 512).await {
                                    Ok(resp) => {
-                                        let sql = resp.text.trim().to_string();
+                                        let sql = clean_sql(&resp.text);
                                        generated_sql.set(Some(sql.clone()));
                                        step.set("running query...".into());
                                        let query_result = run_sql(&sql).await;
                                        if let Err(ref err) = query_result {
                                            if err.contains("error") {
                                                step.set("fixing SQL...".into());
                                                let retry_prompt = format!(
                                                    "The SQL you wrote had an error:\n{err}\n\n{schema_ctx}\n\nOriginal question: {q}\n\nWrite a CORRECTED SQL query using ONLY the columns listed. Output ONLY SQL."
                                                );
                                                if let Ok(rr) = ai_generate(&retry_prompt, 512).await {
                                                    let rsql = clean_sql(&rr.text);
                                                    generated_sql.set(Some(rsql.clone()));
                                                    step.set("running corrected query...".into());
                                                    result.set(Some(run_sql(&rsql).await));
                                                } else {
                                                    result.set(Some(query_result));
                                                }
                                            } else {
                                                result.set(Some(query_result));
                                            }
                                        } else {
                                            result.set(Some(query_result));
                                        }
                                    }
                                    Err(e) => {
                                        result.set(Some(Err(format!("AI error: {e}"))));
                                    }
@@ -532,6 +633,303 @@ fn SqlPanel() -> Element {
    }
 }
 // === DASHBOARD — System overview ===
 #[component]
 fn DashboardPanel() -> Element {
    let mut stats = use_signal(|| None::<serde_json::Value>);
    let mut loading = use_signal(|| false);
    let do_load = move || {
        spawn(async move {
            loading.set(true);
            let client = reqwest::Client::new();
            let datasets = fetch_datasets().await.ok().map(|d| d.len()).unwrap_or(0);
            let row_count = run_sql("SELECT (SELECT COUNT(*) FROM candidates) + (SELECT COUNT(*) FROM clients) + (SELECT COUNT(*) FROM job_orders) + (SELECT COUNT(*) FROM placements) + (SELECT COUNT(*) FROM timesheets) + (SELECT COUNT(*) FROM call_log) + (SELECT COUNT(*) FROM email_log) as total").await.ok().and_then(|r| r.rows.as_array()?.first()?.get("total")?.as_i64()).unwrap_or(0);
            let cache: Option<serde_json::Value> = match client.get(&format!("{}/query/cache/stats", api_base())).send().await {
                Ok(r) => r.json().await.ok(),
                Err(_) => None,
            };
            let indexes: Vec<serde_json::Value> = match client.get(&format!("{}/vectors/indexes", api_base())).send().await {
                Ok(r) => r.json().await.unwrap_or_default(),
                Err(_) => vec![],
            };
            let tools: Vec<serde_json::Value> = match client.get(&format!("{}/tools", api_base())).send().await {
                Ok(r) => r.json().await.unwrap_or_default(),
                Err(_) => vec![],
            };
            let jobs: Vec<serde_json::Value> = match client.get(&format!("{}/vectors/jobs", api_base())).send().await {
                Ok(r) => r.json().await.unwrap_or_default(),
                Err(_) => vec![],
            };
            let hnsw: Vec<String> = match client.get(&format!("{}/vectors/hnsw/list", api_base())).send().await {
                Ok(r) => r.json().await.unwrap_or_default(),
                Err(_) => vec![],
            };
            let journal: Option<serde_json::Value> = match client.get(&format!("{}/journal/stats", api_base())).send().await {
                Ok(r) => r.json().await.ok(),
                Err(_) => None,
            };
            stats.set(Some(serde_json::json!({
                "datasets": datasets,
                "total_rows": row_count,
                "cache": cache,
                "vector_indexes": indexes.len(),
                "vector_total_chunks": indexes.iter().filter_map(|i| i.get("chunk_count")?.as_i64()).sum::<i64>(),
                "hnsw_loaded": hnsw.len(),
                "tools": tools.len(),
                "jobs_total": jobs.len(),
                "jobs_running": jobs.iter().filter(|j| j.get("status").and_then(|s| s.as_str()) == Some("running")).count(),
                "journal": journal,
            })));
            loading.set(false);
        });
    };
    // Auto-load on mount
    // Load on first render (user clicks Dashboard tab)
    let mut loaded = use_signal(|| false);
    use_effect(move || {
        if !*loaded.read() {
            loaded.set(true);
            do_load();
        }
    });
    rsx! {
        div { class: "panel",
            div { class: "dashboard-hero",
                h2 { "Lakehouse" }
                p { class: "subtitle", "Rust-first data platform — SQL + AI + RAG over object storage" }
            }
            if let Some(s) = stats.read().as_ref() {
                div { class: "stat-grid",
                    div { class: "stat-card",
                        div { class: "stat-value", "{s[\"datasets\"]}" }
                        div { class: "stat-label", "Datasets" }
                    }
                    div { class: "stat-card",
                        div { class: "stat-value", {format!("{:.1}M", s["total_rows"].as_i64().unwrap_or(0) as f64 / 1_000_000.0)} }
                        div { class: "stat-label", "Total Rows" }
                    }
                    div { class: "stat-card",
                        div { class: "stat-value", {format!("{}K", s["vector_total_chunks"].as_i64().unwrap_or(0) / 1000)} }
                        div { class: "stat-label", "Embeddings" }
                    }
                    div { class: "stat-card accent",
                        div { class: "stat-value", "{s[\"hnsw_loaded\"]}" }
                        div { class: "stat-label", "HNSW Indexes (27ms)" }
                    }
                    div { class: "stat-card",
                        div { class: "stat-value", "{s[\"tools\"]}" }
                        div { class: "stat-label", "Governed Tools" }
                    }
                    div { class: "stat-card",
                        div { class: "stat-value", {
                            s.get("cache")
                                .and_then(|c| c.get("datasets"))
                                .and_then(|d| d.as_i64())
                                .map(|n| format!("{n}"))
                                .unwrap_or("0".into())
                        } }
                        div { class: "stat-label", "Cached Datasets" }
                    }
                }
                div { class: "architecture-section",
                    h3 { "Architecture" }
                    div { class: "arch-grid",
                        div { class: "arch-card",
                            div { class: "arch-title", "Ingest" }
                            div { class: "arch-items", "CSV, JSON, PDF, Text, PostgreSQL, File Watcher" }
                        }
                        div { class: "arch-card",
                            div { class: "arch-title", "Storage" }
                            div { class: "arch-items", "Parquet on Object Storage, Delta Writes, Compaction" }
                        }
                        div { class: "arch-card",
                            div { class: "arch-title", "Query" }
                            div { class: "arch-items", "DataFusion SQL, MemCache (9.8x), Hot/Cold" }
                        }
                        div { class: "arch-card",
                            div { class: "arch-title", "AI" }
                            div { class: "arch-items", "Ollama (local), Embed, Generate, RAG, HNSW" }
                        }
                        div { class: "arch-card",
                            div { class: "arch-title", "Governance" }
                            div { class: "arch-items", "Event Journal, PII Detection, Tool Registry, Access Control" }
                        }
                        div { class: "arch-card",
                            div { class: "arch-title", "Agents" }
                            div { class: "arch-items", "Workspaces, Handoff, Shortlists, Activity Logs" }
                        }
                    }
                }
                div { class: "phases-section",
                    h3 { "Build Progression" }
                    div { class: "phase-list",
                        {rsx! {
                            PhaseItem { num: "0-5", name: "Foundation", detail: "Storage, Catalog, DataFusion, AI, UI, gRPC" }
                            PhaseItem { num: "6", name: "Ingest Pipeline", detail: "CSV/JSON/PDF/Text auto-schema" }
                            PhaseItem { num: "7", name: "Vector + RAG", detail: "Embed, Search, LLM Answers" }
                            PhaseItem { num: "8", name: "Hot Cache", detail: "9.8x speedup, Delta Writes" }
                            PhaseItem { num: "8.5", name: "Agent Workspaces", detail: "Per-contract, Instant Handoff" }
                            PhaseItem { num: "9", name: "Event Journal", detail: "Append-only Mutation History" }
                            PhaseItem { num: "10", name: "Rich Catalog", detail: "PII Detection, Lineage" }
                            PhaseItem { num: "11", name: "Embedding Versioning", detail: "Model-proof Vectors" }
                            PhaseItem { num: "12", name: "Tool Registry", detail: "6 Governed Actions + Audit" }
                            PhaseItem { num: "13", name: "Access Control", detail: "Role-based, Field-level" }
                            PhaseItem { num: "14", name: "Schema Evolution", detail: "Diff Detection, AI Migration" }
                            PhaseItem { num: "15", name: "HNSW Index", detail: "100K Search in 27ms" }
                            PhaseItem { num: "16", name: "File Watcher", detail: "Auto-ingest from Inbox" }
                            PhaseItem { num: "17", name: "DB Connector", detail: "PostgreSQL Import" }
                        }}
                    }
                }
            } else if *loading.read() {
                div { class: "loading", "loading dashboard..." }
            }
            button { class: "btn", onclick: move |_| do_load(), "Refresh" }
        }
    }
 }
 #[component]
 fn PhaseItem(num: String, name: String, detail: String) -> Element {
    rsx! {
        div { class: "phase-item",
            span { class: "phase-num", "{num}" }
            span { class: "phase-name", "{name}" }
            span { class: "phase-detail", "{detail}" }
        }
    }
 }
 // === INGEST — Data on-ramp ===
 #[component]
 fn IngestPanel() -> Element {
    let mut pg_host = use_signal(|| "127.0.0.1".to_string());
    let mut pg_db = use_signal(|| "knowledge_base".to_string());
    let mut pg_tables = use_signal(|| None::<Vec<String>>);
    let mut pg_result = use_signal(|| None::<Result<serde_json::Value, String>>);
    let mut pg_loading = use_signal(|| false);
    let list_tables = move |_| {
        let host = pg_host.read().clone();
        let db = pg_db.read().clone();
        spawn(async move {
            pg_loading.set(true);
            let client = reqwest::Client::new();
            let resp = client.post(&format!("{}/ingest/postgres/tables", api_base()))
                .json(&serde_json::json!({"host": host, "port": 5432, "database": db, "user": "postgres", "password": ""}))
                .send().await;
            match resp {
                Ok(r) => {
                    if let Ok(tables) = r.json::<Vec<String>>().await {
                        pg_tables.set(Some(tables));
                    }
                }
                Err(e) => pg_tables.set(None),
            }
            pg_loading.set(false);
        });
    };
    let mut import_table = move |table: String| {
        let host = pg_host.read().clone();
        let db = pg_db.read().clone();
        spawn(async move {
            pg_result.set(None);
            let client = reqwest::Client::new();
            let resp = client.post(&format!("{}/ingest/postgres/import", api_base()))
                .json(&serde_json::json!({"host": host, "port": 5432, "database": db, "user": "postgres", "password": "", "table": table}))
                .send().await;
            match resp {
                Ok(r) => {
                    match r.json::<serde_json::Value>().await {
                        Ok(v) => pg_result.set(Some(Ok(v))),
                        Err(e) => pg_result.set(Some(Err(e.to_string()))),
                    }
                }
                Err(e) => pg_result.set(Some(Err(e.to_string()))),
            }
        });
    };
    rsx! {
        div { class: "panel",
            h2 { "Data Ingest" }
            p { class: "subtitle", "Bring data in from files, databases, or the auto-watch inbox" }
            div { class: "panel-section",
                h3 { "File Upload" }
                p { class: "hint", "POST a file to /ingest/file — or drop it in ./inbox/ for auto-ingest" }
                div { class: "arch-card",
                    div { class: "arch-title", "Supported Formats" }
                    div { class: "arch-items", "CSV (auto-schema) | JSON (nested flattening) | PDF (text extraction) | Text/SMS" }
                }
                div { class: "arch-card",
                    div { class: "arch-title", "Auto-Watch Inbox" }
                    div { class: "arch-items", "Drop files in ./inbox/ → auto-detected → Parquet → queryable in <15s" }
                }
            }
            div { class: "panel-section",
                h3 { "PostgreSQL Import" }
                div { class: "form-row",
                    label { "Host" }
                    input { value: "{pg_host}", oninput: move |e| pg_host.set(e.value()) }
                }
                div { class: "form-row",
                    label { "Database" }
                    input { value: "{pg_db}", oninput: move |e| pg_db.set(e.value()) }
                }
                button { class: "btn", disabled: *pg_loading.read(), onclick: list_tables, "List Tables" }
                if let Some(tables) = pg_tables.read().as_ref() {
                    div { class: "table-list",
                        for table in tables.iter() {
                            {
                                let t = table.clone();
                                let t2 = table.clone();
                                rsx! {
                                    div { class: "table-item",
                                        span { "{t}" }
                                        button { class: "btn btn-sm", onclick: move |_| import_table(t2.clone()), "Import" }
                                    }
                                }
                            }
                        }
                    }
                }
                if let Some(result) = pg_result.read().as_ref() {
                    match result {
                        Ok(v) => rsx! {
                            div { class: "result-box",
                                pre { {serde_json::to_string_pretty(v).unwrap_or_default()} }
                            }
                        },
                        Err(e) => rsx! { div { class: "error", "{e}" } },
                    }
                }
            }
        }
    }
 }
 // === STATUS — System health ===
 #[component]
@@ -633,6 +1031,9 @@ fn ResultsTable(response: QueryResponse) -> Element {
        if response.row_count == 0 {
            div { class: "empty-sm", "no rows returned" }
        } else if let Some(rows) = rows {
            if rows.len() > 500 {
                div { class: "results-info", "Showing first 500 of {response.row_count} rows (use SQL tab with LIMIT for larger)" }
            }
            div { class: "table-wrap",
                table {
                    thead {
@@ -643,7 +1044,7 @@ fn ResultsTable(response: QueryResponse) -> Element {
                        }
                    }
                    tbody {
-                        for row in rows.iter() {
+                        for row in rows.iter().take(500) {
                            tr {
                                for col in response.columns.iter() {
                                    td { {format_cell(row.get(&col.name))} }
@@ -657,6 +1058,49 @@ fn ResultsTable(response: QueryResponse) -> Element {
    }
 }
 /// Clean AI-generated SQL: extract only the SQL query, strip everything else.
 fn clean_sql(raw: &str) -> String {
    let s = raw.trim();
    // Strategy 1: If there's a ```sql...``` block, extract just that
    if let Some(start) = s.find("```sql") {
        let after = &s[start + 6..];
        if let Some(end) = after.find("```") {
            return after[..end].trim().to_string();
        }
    }
    if let Some(start) = s.find("```") {
        let after = &s[start + 3..];
        if let Some(end) = after.find("```") {
            let inner = after[..end].trim();
            // Skip leading "sql" keyword
            let inner = inner.strip_prefix("sql").map(|s| s.trim_start()).unwrap_or(inner);
            return inner.to_string();
        }
    }
    // Strategy 2: Find the first SELECT/WITH/INSERT/UPDATE/DELETE statement
    let upper = s.to_uppercase();
    for keyword in &["SELECT", "WITH", "INSERT", "UPDATE", "DELETE"] {
        if let Some(pos) = upper.find(keyword) {
            let sql_part = &s[pos..];
            // Take up to the first semicolon (or end)
            let end = sql_part.find(';').map(|p| p + 1).unwrap_or(sql_part.len());
            return sql_part[..end].trim().to_string();
        }
    }
    // Strategy 3: Strip leading "sql" and clean up
    let mut result = s.to_string();
    let lines: Vec<&str> = result.lines().collect();
    if let Some(first) = lines.first() {
        if first.trim().eq_ignore_ascii_case("sql") {
            result = lines[1..].join("\n").trim().to_string();
        }
    }
    result
 }
 fn format_cell(val: Option<&serde_json::Value>) -> String {
    match val {
        None | Some(serde_json::Value::Null) => "—".to_string(),
@@ -17,3 +17,4 @@ object_store = { workspace = true }
 parquet = { workspace = true }
 arrow = { workspace = true }
 chrono = { workspace = true }
 instant-distance = { workspace = true }
@@ -40,48 +40,45 @@ pub fn chunk_text(
        }];
    }
    // Work with char indices, not byte indices, to handle Unicode safely
    let chars: Vec<char> = text.chars().collect();
    let char_len = chars.len();
    let mut chunks = Vec::new();
-    let mut start = 0;
+    let mut start = 0usize; // char index
    let mut idx = 0u32;
-    while start < text.len() {
+    while start < char_len {
-        let end = (start + chunk_size).min(text.len());
+        let end = (start + chunk_size).min(char_len);
        let chunk_str: String = chars[start..end].iter().collect();
-        // Try to break at a sentence or paragraph boundary
+        let actual_end = if end < char_len {
-        let chunk_text = &text[start..end];
+            // Look for last sentence boundary
-        let actual_end = if end < text.len() {
+            if let Some(pos) = chunk_str.rfind(". ") {
-            // Look for last sentence boundary in the chunk
+                start + chunk_str[..pos + 2].chars().count()
-            if let Some(pos) = chunk_text.rfind(". ") {
+            } else if let Some(pos) = chunk_str.rfind('\n') {
-                start + pos + 2
+                start + chunk_str[..pos + 1].chars().count()
-            } else if let Some(pos) = chunk_text.rfind('\n') {
+            } else if let Some(pos) = chunk_str.rfind(' ') {
-                start + pos + 1
+                start + chunk_str[..pos + 1].chars().count()
            } else {
                // Fall back to word boundary
                if let Some(pos) = chunk_text.rfind(' ') {
                    start + pos + 1
            } else {
                end
            }
            }
        } else {
            end
        };
-        let chunk = text[start..actual_end].trim();
+        let chunk: String = chars[start..actual_end].iter().collect();
-        if !chunk.is_empty() {
+        let trimmed = chunk.trim();
        if !trimmed.is_empty() {
            chunks.push(TextChunk {
                source: source.to_string(),
                doc_id: doc_id.to_string(),
                chunk_idx: idx,
-                text: chunk.to_string(),
+                text: trimmed.to_string(),
            });
            idx += 1;
        }
-        // Advance with overlap
+        if actual_end >= char_len { break; }
        if actual_end >= text.len() {
            break;
        }
        start = if actual_end > overlap { actual_end - overlap } else { actual_end };
    }
@@ -0,0 +1,168 @@
 /// HNSW (Hierarchical Navigable Small World) index for fast approximate nearest neighbor search.
 /// Wraps instant-distance to provide <50ms search over 100K+ vectors.
 /// Falls back to brute-force for small datasets.
 use instant_distance::{Builder, HnswMap, Search, Point};
 use std::sync::Arc;
 use tokio::sync::RwLock;
 use crate::store::StoredEmbedding;
 /// A vector point for HNSW — wraps f32 slice with cosine distance.
 #[derive(Clone)]
 struct VecPoint(Vec<f32>);
 impl Point for VecPoint {
    fn distance(&self, other: &Self) -> f32 {
        // Cosine distance = 1 - cosine_similarity
        let dot: f32 = self.0.iter().zip(other.0.iter()).map(|(a, b)| a * b).sum();
        let norm_a: f32 = self.0.iter().map(|x| x * x).sum::<f32>().sqrt();
        let norm_b: f32 = other.0.iter().map(|x| x * x).sum::<f32>().sqrt();
        if norm_a == 0.0 || norm_b == 0.0 {
            return 1.0;
        }
        1.0 - (dot / (norm_a * norm_b))
    }
 }
 /// HNSW search result.
 #[derive(Debug, Clone, serde::Serialize)]
 pub struct HnswResult {
    pub source: String,
    pub doc_id: String,
    pub chunk_idx: u32,
    pub chunk_text: String,
    pub score: f32,  // cosine similarity (1.0 = identical)
 }
 /// An HNSW index built from stored embeddings.
 pub struct HnswIndex {
    map: HnswMap<VecPoint, usize>, // value is index into metadata vec
    metadata: Vec<EmbeddingMeta>,
 }
 #[derive(Clone)]
 struct EmbeddingMeta {
    source: String,
    doc_id: String,
    chunk_idx: u32,
    chunk_text: String,
 }
 /// Shared, thread-safe index store.
 #[derive(Clone)]
 pub struct HnswStore {
    indexes: Arc<RwLock<std::collections::HashMap<String, Arc<HnswIndex>>>>,
 }
 impl HnswStore {
    pub fn new() -> Self {
        Self {
            indexes: Arc::new(RwLock::new(std::collections::HashMap::new())),
        }
    }
    /// Build an HNSW index from stored embeddings.
    pub async fn build_index(
        &self,
        index_name: &str,
        embeddings: Vec<StoredEmbedding>,
    ) -> Result<BuildStats, String> {
        let n = embeddings.len();
        if n == 0 {
            return Err("no embeddings to index".into());
        }
        tracing::info!("building HNSW index '{}' from {} vectors", index_name, n);
        let start = std::time::Instant::now();
        // Separate points and metadata
        let mut points = Vec::with_capacity(n);
        let mut metadata = Vec::with_capacity(n);
        let mut values = Vec::with_capacity(n);
        for (i, emb) in embeddings.into_iter().enumerate() {
            points.push(VecPoint(emb.vector));
            metadata.push(EmbeddingMeta {
                source: emb.source,
                doc_id: emb.doc_id,
                chunk_idx: emb.chunk_idx,
                chunk_text: emb.chunk_text,
            });
            values.push(i);
        }
        // Build HNSW — this is the expensive part
        let map = Builder::default()
            .ef_construction(40)   // balanced for 100K scale
            .ef_search(50)         // fast search with good recall
            .build(points, values);
        let build_time = start.elapsed().as_secs_f32();
        tracing::info!("HNSW index '{}' built: {} vectors in {:.1}s", index_name, n, build_time);
        let index = Arc::new(HnswIndex { map, metadata });
        self.indexes.write().await.insert(index_name.to_string(), index);
        Ok(BuildStats {
            index_name: index_name.to_string(),
            vectors: n,
            build_time_secs: build_time,
        })
    }
    /// Search an HNSW index. Returns approximate nearest neighbors.
    pub async fn search(
        &self,
        index_name: &str,
        query: &[f32],
        top_k: usize,
    ) -> Result<Vec<HnswResult>, String> {
        let indexes = self.indexes.read().await;
        let index = indexes.get(index_name)
            .ok_or_else(|| format!("HNSW index not found: {index_name}"))?;
        let query_point = VecPoint(query.to_vec());
        let mut search = Search::default();
        let results = index.map.search(&query_point, &mut search);
        let mut out = Vec::with_capacity(top_k);
        for item in results.take(top_k) {
            let meta_idx = *item.value;
            let meta = &index.metadata[meta_idx];
            // Convert distance back to similarity
            let similarity = 1.0 - item.distance;
            out.push(HnswResult {
                source: meta.source.clone(),
                doc_id: meta.doc_id.clone(),
                chunk_idx: meta.chunk_idx,
                chunk_text: meta.chunk_text.clone(),
                score: similarity,
            });
        }
        Ok(out)
    }
    /// Check if an index exists.
    pub async fn has_index(&self, name: &str) -> bool {
        self.indexes.read().await.contains_key(name)
    }
    /// List all loaded indexes.
    pub async fn list(&self) -> Vec<String> {
        self.indexes.read().await.keys().cloned().collect()
    }
    /// Drop an index from memory.
    pub async fn drop(&self, name: &str) -> bool {
        self.indexes.write().await.remove(name).is_some()
    }
 }
 #[derive(Debug, Clone, serde::Serialize)]
 pub struct BuildStats {
    pub index_name: String,
    pub vectors: usize,
    pub build_time_secs: f32,
 }
@@ -0,0 +1,112 @@
 /// Vector index registry — tracks all indexes with model versioning.
 /// Each index knows which model created it, enabling:
 /// - Multi-version indexes (same data, different models, coexist)
 /// - Incremental re-embed (only new/changed docs on model upgrade)
 /// - A/B search comparison between model versions
 use chrono::{DateTime, Utc};
 use object_store::ObjectStore;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 use storaged::ops;
 /// Metadata for a vector index.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IndexMeta {
    pub index_name: String,
    pub source: String,           // dataset this was built from
    pub model_name: String,       // "nomic-embed-text"
    pub model_version: String,    // "latest" or specific version
    pub dimensions: u32,          // 768
    pub chunk_count: usize,
    pub doc_count: usize,
    pub chunk_size: usize,
    pub overlap: usize,
    pub storage_key: String,      // "vectors/resumes_v1_nomic.parquet"
    pub created_at: DateTime<Utc>,
    pub build_time_secs: f32,
    pub chunks_per_sec: f32,
 }
 /// Registry of all vector indexes.
 #[derive(Clone)]
 pub struct IndexRegistry {
    indexes: Arc<RwLock<HashMap<String, IndexMeta>>>,
    store: Arc<dyn ObjectStore>,
 }
 impl IndexRegistry {
    pub fn new(store: Arc<dyn ObjectStore>) -> Self {
        Self {
            indexes: Arc::new(RwLock::new(HashMap::new())),
            store,
        }
    }
    /// Rebuild from persisted index metadata on startup.
    pub async fn rebuild(&self) -> Result<usize, String> {
        let keys = ops::list(&self.store, Some("vectors/meta/")).await?;
        let mut reg = self.indexes.write().await;
        reg.clear();
        for key in &keys {
            if !key.ends_with(".json") { continue; }
            let data = ops::get(&self.store, key).await?;
            match serde_json::from_slice::<IndexMeta>(&data) {
                Ok(meta) => { reg.insert(meta.index_name.clone(), meta); }
                Err(e) => tracing::warn!("failed to load index meta {key}: {e}"),
            }
        }
        let count = reg.len();
        if count > 0 {
            tracing::info!("loaded {count} vector index metadata entries");
        }
        Ok(count)
    }
    /// Register a new index.
    pub async fn register(&self, meta: IndexMeta) -> Result<(), String> {
        let key = format!("vectors/meta/{}.json", meta.index_name);
        let json = serde_json::to_vec_pretty(&meta).map_err(|e| e.to_string())?;
        ops::put(&self.store, &key, json.into()).await?;
        self.indexes.write().await.insert(meta.index_name.clone(), meta);
        Ok(())
    }
    /// Get metadata for an index.
    pub async fn get(&self, index_name: &str) -> Option<IndexMeta> {
        self.indexes.read().await.get(index_name).cloned()
    }
    /// List all indexes, optionally filtered by source or model.
    pub async fn list(&self, source: Option<&str>, model: Option<&str>) -> Vec<IndexMeta> {
        self.indexes.read().await.values()
            .filter(|m| source.map_or(true, |s| m.source == s))
            .filter(|m| model.map_or(true, |mo| m.model_name == mo))
            .cloned()
            .collect()
    }
    /// Find all versions of an index for a given source dataset.
    /// Returns indexes sorted by creation time (newest first).
    pub async fn versions_for_source(&self, source: &str) -> Vec<IndexMeta> {
        let mut versions: Vec<IndexMeta> = self.indexes.read().await.values()
            .filter(|m| m.source == source)
            .cloned()
            .collect();
        versions.sort_by(|a, b| b.created_at.cmp(&a.created_at));
        versions
    }
    /// Delete an index (metadata only — vector Parquet stays for safety).
    pub async fn delete(&self, index_name: &str) -> Result<(), String> {
        let key = format!("vectors/meta/{index_name}.json");
        ops::delete(&self.store, &key).await?;
        self.indexes.write().await.remove(index_name);
        Ok(())
    }
 }
@@ -1,4 +1,6 @@
 pub mod chunker;
 pub mod hnsw;
 pub mod index_registry;
 pub mod jobs;
 pub mod store;
 pub mod search;
@@ -1,6 +1,6 @@
 use axum::{
    Json, Router,
-    extract::{Path, State},
+    extract::{Path, Query, State},
    http::StatusCode,
    response::IntoResponse,
    routing::{get, post},
@@ -10,23 +10,30 @@ use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use aibridge::client::{AiClient, EmbedRequest};
-use crate::{chunker, jobs, rag, search, store, supervisor};
+use crate::{chunker, hnsw, index_registry, jobs, rag, search, store, supervisor};
 #[derive(Clone)]
 pub struct VectorState {
    pub store: Arc<dyn ObjectStore>,
    pub ai_client: AiClient,
    pub job_tracker: jobs::JobTracker,
    pub index_registry: index_registry::IndexRegistry,
    pub hnsw_store: hnsw::HnswStore,
 }
 pub fn router(state: VectorState) -> Router {
    Router::new()
        .route("/health", get(health))
        .route("/index", post(create_index))
        .route("/indexes", get(list_indexes))
        .route("/indexes/{name}", get(get_index_meta))
        .route("/jobs", get(list_jobs))
        .route("/jobs/{id}", get(get_job))
        .route("/search", post(search_index))
        .route("/rag", post(rag_query))
        .route("/hnsw/build", post(build_hnsw))
        .route("/hnsw/search", post(search_hnsw))
        .route("/hnsw/list", get(list_hnsw))
        .with_state(state)
 }
@@ -88,17 +95,42 @@ async fn create_index(
    let tracker = state.job_tracker.clone();
    let ai_client = state.ai_client.clone();
    let obj_store = state.store.clone();
    let registry = state.index_registry.clone();
    let jid = job_id.clone();
    let source_name = req.source.clone();
    let idx_name = req.index_name.clone();
    tokio::spawn(async move {
        let start_time = std::time::Instant::now();
        let config = supervisor::SupervisorConfig::default();
        let result = supervisor::run_supervised(
-            &jid, &index_name, chunks, &ai_client, &obj_store, &tracker, config,
+            &jid, &idx_name, chunks, &ai_client, &obj_store, &tracker, config,
        ).await;
        match result {
            Ok(key) => {
                let elapsed = start_time.elapsed().as_secs_f32();
                let rate = if elapsed > 0.0 { n_chunks as f32 / elapsed } else { 0.0 };
                // Register index metadata with model version info
                let meta = index_registry::IndexMeta {
                    index_name: idx_name.clone(),
                    source: source_name,
                    model_name: "nomic-embed-text".to_string(), // from sidecar config
                    model_version: "latest".to_string(),
                    dimensions: 768,
                    chunk_count: n_chunks,
                    doc_count: n_docs,
                    chunk_size: chunk_size,
                    overlap: overlap,
                    storage_key: key.clone(),
                    created_at: chrono::Utc::now(),
                    build_time_secs: elapsed,
                    chunks_per_sec: rate,
                };
                let _ = registry.register(meta).await;
                tracker.complete(&jid, key).await;
-                tracing::info!("job {jid}: completed");
+                tracing::info!("job {jid}: completed — {n_chunks} chunks in {elapsed:.0}s ({rate:.0}/sec)");
            }
            Err(e) => {
                tracker.fail(&jid, e.clone()).await;
@@ -116,8 +148,37 @@ async fn create_index(
    })))
 }
-/// Run the actual embedding work in background.
+// --- Index Registry ---
-async fn run_embedding_job(
+
 #[derive(Deserialize)]
 struct IndexListQuery {
    source: Option<String>,
    model: Option<String>,
 }
 async fn list_indexes(
    State(state): State<VectorState>,
    Query(q): Query<IndexListQuery>,
 ) -> impl IntoResponse {
    let indexes = state.index_registry.list(q.source.as_deref(), q.model.as_deref()).await;
    Json(indexes)
 }
 async fn get_index_meta(
    State(state): State<VectorState>,
    Path(name): Path<String>,
 ) -> impl IntoResponse {
    match state.index_registry.get(&name).await {
        Some(meta) => Ok(Json(meta)),
        None => Err((StatusCode::NOT_FOUND, format!("index not found: {name}"))),
    }
 }
 // --- unused legacy function below, kept for reference ---
 #[allow(dead_code)]
 /// Legacy single-pipeline embedding (replaced by supervisor).
 async fn _run_embedding_job_legacy(
    job_id: &str,
    index_name: &str,
    chunks: &[chunker::TextChunk],
@@ -239,3 +300,75 @@ async fn rag_query(
        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
    }
 }
 // --- HNSW Fast Search ---
 #[derive(Deserialize)]
 struct BuildHnswRequest {
    /// Name of the stored vector index to build HNSW from
    index_name: String,
 }
 /// Build an HNSW index from an existing stored vector index.
 /// Loads embeddings from Parquet, builds HNSW in memory.
 async fn build_hnsw(
    State(state): State<VectorState>,
    Json(req): Json<BuildHnswRequest>,
 ) -> impl IntoResponse {
    tracing::info!("building HNSW for '{}'", req.index_name);
    // Load embeddings from Parquet
    let embeddings = store::load_embeddings(&state.store, &req.index_name)
        .await
        .map_err(|e| (StatusCode::NOT_FOUND, format!("index not found: {e}")))?;
    let n = embeddings.len();
    tracing::info!("loaded {} embeddings, building HNSW...", n);
    // Build HNSW
    match state.hnsw_store.build_index(&req.index_name, embeddings).await {
        Ok(stats) => Ok(Json(stats)),
        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
    }
 }
 #[derive(Deserialize)]
 struct HnswSearchRequest {
    index_name: String,
    query: String,
    top_k: Option<usize>,
 }
 /// Search using HNSW — approximate nearest neighbors, much faster than brute-force.
 async fn search_hnsw(
    State(state): State<VectorState>,
    Json(req): Json<HnswSearchRequest>,
 ) -> impl IntoResponse {
    let top_k = req.top_k.unwrap_or(5);
    // Embed query
    let embed_resp = state.ai_client.embed(EmbedRequest {
        texts: vec![req.query.clone()],
        model: None,
    }).await.map_err(|e| (StatusCode::BAD_GATEWAY, format!("embed error: {e}")))?;
    if embed_resp.embeddings.is_empty() {
        return Err((StatusCode::BAD_GATEWAY, "no embedding returned".to_string()));
    }
    let query_vec: Vec<f32> = embed_resp.embeddings[0].iter().map(|&x| x as f32).collect();
    // Search HNSW
    match state.hnsw_store.search(&req.index_name, &query_vec, top_k).await {
        Ok(results) => Ok(Json(serde_json::json!({
            "results": results,
            "query": req.query,
            "method": "hnsw",
        }))),
        Err(e) => Err((StatusCode::NOT_FOUND, e)),
    }
 }
 async fn list_hnsw(State(state): State<VectorState>) -> impl IntoResponse {
    Json(state.hnsw_store.list().await)
 }
@@ -1,15 +0,0 @@
 {
  "id": "03b65605-7cce-4a49-b338-4f19b0ff2ed5",
  "name": "call_log",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/call_log.parquet",
      "size_bytes": 35951077,
      "created_at": "2026-03-27T14:00:44.377704982Z"
    }
  ],
  "created_at": "2026-03-27T14:00:44.377712082Z",
  "updated_at": "2026-03-27T14:00:44.377712082Z"
 }
@@ -0,0 +1,72 @@
 {
  "id": "0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1",
  "name": "rust_in_tech_stack",
  "schema_fingerprint": "314ee8ede68287240d7cec1403d252787c8ceb4ee2afa72cffcb0853ac3ac889",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/rust_in_tech_stack.parquet",
      "size_bytes": 99623,
      "created_at": "2026-03-28T03:32:50.769271381Z"
    }
  ],
  "created_at": "2026-03-28T03:32:50.769272195Z",
  "updated_at": "2026-03-28T03:32:50.769710478Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "Rust in Tech Stack.pdf",
    "ingest_job": "ingest-1774668770769",
    "ingest_timestamp": "2026-03-28T03:32:50.769271381Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 49
 }
@@ -1,15 +0,0 @@
 {
  "id": "0e4feb1a-1421-46ac-8222-ba0f0bd6e13e",
  "name": "email_log",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/email_log.parquet",
      "size_bytes": 16768671,
      "created_at": "2026-03-27T14:00:46.272499334Z"
    }
  ],
  "created_at": "2026-03-27T14:00:46.272507485Z",
  "updated_at": "2026-03-27T14:00:46.272507485Z"
 }
@@ -0,0 +1,23 @@
 {
  "id": "0fd78303-9ad4-45fd-90d7-db95607d9ab1",
  "name": "timesheets",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/timesheets.parquet",
      "size_bytes": 17539932,
      "created_at": "2026-03-27T14:42:43.922019299Z"
    }
  ],
  "created_at": "2026-03-27T14:42:43.922025703Z",
  "updated_at": "2026-03-27T14:42:43.922025703Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "1339f3d6-7677-47fb-8182-5f8e43f27cde",
  "name": "job_orders",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/job_orders.parquet",
      "size_bytes": 905534,
      "created_at": "2026-03-27T14:42:38.935718195Z"
    }
  ],
  "created_at": "2026-03-27T14:42:38.935724058Z",
  "updated_at": "2026-03-27T14:42:38.935724058Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -1,15 +0,0 @@
 {
  "id": "154cb8fe-5dcb-4d23-8ddb-c95b259757e9",
  "name": "timesheets",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/timesheets.parquet",
      "size_bytes": 17539932,
      "created_at": "2026-03-27T14:00:40.845373500Z"
    }
  ],
  "created_at": "2026-03-27T14:00:40.845380446Z",
  "updated_at": "2026-03-27T14:00:40.845380446Z"
 }
@@ -0,0 +1,23 @@
 {
  "id": "1d8a065e-59c1-45ce-967b-398bc8370cbb",
  "name": "candidates",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/candidates.parquet",
      "size_bytes": 10592165,
      "created_at": "2026-03-27T14:42:38.823368759Z"
    }
  ],
  "created_at": "2026-03-27T14:42:38.823374843Z",
  "updated_at": "2026-03-27T14:42:38.823374843Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,58 @@
 {
  "id": "24a05512-dc06-4884-9889-2ea978abdaec",
  "name": "processed_extractions",
  "schema_fingerprint": "c2853ba4c0cc5a897b37680fefea2708f51d51774226b00b72444e79d594c874",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/processed_extractions.parquet",
      "size_bytes": 28175,
      "created_at": "2026-03-28T05:16:57.437041979Z"
    }
  ],
  "created_at": "2026-03-28T05:16:57.437042648Z",
  "updated_at": "2026-03-28T05:16:57.437403737Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "conversation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "type",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "processed_extractions.csv",
    "ingest_job": "ingest-1774675017437",
    "ingest_timestamp": "2026-03-28T05:16:57.437041979Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 543
 }
@@ -0,0 +1,65 @@
 {
  "id": "3b12ae24-17d4-4325-92bf-f3155982f3bf",
  "name": "users",
  "schema_fingerprint": "30c0e31f0963e6f4af02131bbb9ea246fbbd068b849b833565a4b28211fbc90b",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/users.parquet",
      "size_bytes": 2012,
      "created_at": "2026-03-28T01:38:59.904968123Z"
    }
  ],
  "created_at": "2026-03-28T01:38:59.904968680Z",
  "updated_at": "2026-03-28T01:38:59.905221340Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "id",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "username",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "password_hash",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "created_at",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "postgresql",
    "source_file": "127.0.0.1:5432/knowledge_base.users",
    "ingest_job": "pg-import-1774661939904",
    "ingest_timestamp": "2026-03-28T01:38:59.904968123Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 1
 }
@@ -0,0 +1,93 @@
 {
  "id": "3c2579b4-f3f3-4875-95fa-58d8b49ad94c",
  "name": "meta_runs",
  "schema_fingerprint": "68f2c0d7a3ceb0aaa3c17c64900704519c72d213161bc9e5179c42ee53f6d0df",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/meta_runs.parquet",
      "size_bytes": 729773,
      "created_at": "2026-03-28T01:38:57.380576453Z"
    }
  ],
  "created_at": "2026-03-28T01:38:57.380577270Z",
  "updated_at": "2026-03-28T01:38:57.380846224Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "id",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "pipeline_id",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "iteration",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "stage_results",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "final_output",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "score",
      "data_type": "Float64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "model_config",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "duration_ms",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "created_at",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "postgresql",
    "source_file": "127.0.0.1:5432/knowledge_base.meta_runs",
    "ingest_job": "pg-import-1774661937380",
    "ingest_timestamp": "2026-03-28T01:38:57.380576453Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 38
 }
@@ -0,0 +1,58 @@
 {
  "id": "3ead1a7a-810b-49d1-add8-752bab52dfae",
  "name": "processed_validated",
  "schema_fingerprint": "8078de214059a2a4582203b32c1172fcc311c2d3c762a3b075c07268308f1841",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/processed_validated.parquet",
      "size_bytes": 15271,
      "created_at": "2026-03-28T05:16:57.439472748Z"
    }
  ],
  "created_at": "2026-03-28T05:16:57.439473552Z",
  "updated_at": "2026-03-28T05:16:57.439749766Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "conversation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "type",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "processed_validated.csv",
    "ingest_job": "ingest-1774675017439",
    "ingest_timestamp": "2026-03-28T05:16:57.439472748Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 60
 }
@@ -0,0 +1,72 @@
 {
  "id": "54cc1ae9-14fe-40f6-88e4-7c623f57b577",
  "name": "crm_development_options",
  "schema_fingerprint": "bb04328fa99161403fa0993f59bf14adcf08eb551c7b0c5e9b9e305e5e0c8556",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/crm_development_options.parquet",
      "size_bytes": 50528,
      "created_at": "2026-03-28T03:30:29.036504470Z"
    }
  ],
  "created_at": "2026-03-28T03:30:29.036504841Z",
  "updated_at": "2026-03-28T03:30:29.036642992Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "CRM Development Options.pdf",
    "ingest_job": "ingest-1774668629036",
    "ingest_timestamp": "2026-03-28T03:30:29.036504470Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 25
 }
@@ -0,0 +1,72 @@
 {
  "id": "5d4bbdb1-fb53-4cea-86d5-309fe067111c",
  "name": "legacy_staffing_crm_search",
  "schema_fingerprint": "87b3fa58cae74468bf7fbbe32397b1a88bc281832bb0226b9eae5ce2501cda4e",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/legacy_staffing_crm_search.parquet",
      "size_bytes": 2722188,
      "created_at": "2026-03-28T03:34:14.444384147Z"
    }
  ],
  "created_at": "2026-03-28T03:34:14.444384964Z",
  "updated_at": "2026-03-28T03:34:14.444631183Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "Legacy Staffing CRM Search.pdf",
    "ingest_job": "ingest-1774668854444",
    "ingest_timestamp": "2026-03-28T03:34:14.444384147Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 1
 }
@@ -0,0 +1,51 @@
 {
  "id": "667478c3-47f9-4076-bab2-8293999d4db2",
  "name": "knowledge_insights",
  "schema_fingerprint": "36896987c7df0e78f5282208e3b8bbbaf9c017588d0bb42a20a7dbe97d8d4fa0",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/knowledge_insights.parquet",
      "size_bytes": 13099,
      "created_at": "2026-03-28T04:06:05.361963023Z"
    }
  ],
  "created_at": "2026-03-28T04:06:05.361963273Z",
  "updated_at": "2026-03-28T04:06:05.362127023Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "conversation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "insight",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "knowledge_insights.csv",
    "ingest_job": "ingest-1774670765361",
    "ingest_timestamp": "2026-03-28T04:06:05.361963023Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 100
 }
@@ -0,0 +1,51 @@
 {
  "id": "799c825c-950e-480e-a38e-4407377e43e3",
  "name": "knowledge_topics",
  "schema_fingerprint": "356c37c8fef56ce7f7882c80f1301cc2f70bcbf874719cb0628b42179874494c",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/knowledge_topics.parquet",
      "size_bytes": 14252,
      "created_at": "2026-03-28T04:06:05.358571822Z"
    }
  ],
  "created_at": "2026-03-28T04:06:05.358572461Z",
  "updated_at": "2026-03-28T04:06:05.358833280Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "conversation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "topic",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "knowledge_topics.csv",
    "ingest_job": "ingest-1774670765358",
    "ingest_timestamp": "2026-03-28T04:06:05.358571822Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 496
 }
@@ -0,0 +1,72 @@
 {
  "id": "7ca19535-838e-4f79-aa6f-d8ebaa2e6edc",
  "name": "code_sorting_assistance",
  "schema_fingerprint": "47e317d38bbad23688f6c274cbecc3dfd7076e42a6a0d09002a0bef835789bc2",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/code_sorting_assistance.parquet",
      "size_bytes": 122034,
      "created_at": "2026-03-28T03:31:08.585681049Z"
    }
  ],
  "created_at": "2026-03-28T03:31:08.585682007Z",
  "updated_at": "2026-03-28T03:31:08.585978498Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "Code Sorting Assistance.pdf",
    "ingest_job": "ingest-1774668668585",
    "ingest_timestamp": "2026-03-28T03:31:08.585681049Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 70
 }
@@ -0,0 +1,51 @@
 {
  "id": "80eaf246-778b-4987-9bef-62a6cccb6021",
  "name": "knowledge_questions",
  "schema_fingerprint": "d86b4ddf7a9694118aeff3d8aa41312e0c3091b0f15cad1079ee9561c29f9c91",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/knowledge_questions.parquet",
      "size_bytes": 17253,
      "created_at": "2026-03-28T04:06:05.363389625Z"
    }
  ],
  "created_at": "2026-03-28T04:06:05.363390263Z",
  "updated_at": "2026-03-28T04:06:05.363439745Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "conversation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "question",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "knowledge_questions.csv",
    "ingest_job": "ingest-1774670765363",
    "ingest_timestamp": "2026-03-28T04:06:05.363389625Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 200
 }
@@ -0,0 +1,72 @@
 {
  "id": "8f5c82bf-93ec-4ce7-b2d1-61e559242add",
  "name": "llms3_update_prompt",
  "schema_fingerprint": "77cfcdb5e22f095302bb816abd4e33380a6b357cd7d23efee5076083020fbf32",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/llms3_update_prompt.parquet",
      "size_bytes": 39396,
      "created_at": "2026-03-28T03:28:04.752204191Z"
    }
  ],
  "created_at": "2026-03-28T03:28:04.752204734Z",
  "updated_at": "2026-03-28T03:28:04.752281926Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "LLMS3 Update Prompt.pdf",
    "ingest_job": "ingest-1774668484752",
    "ingest_timestamp": "2026-03-28T03:28:04.752204191Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 6
 }
@@ -0,0 +1,23 @@
 {
  "id": "94a8bd16-6756-43af-b951-09a9e6b8300f",
  "name": "clients",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/clients.parquet",
      "size_bytes": 21971,
      "created_at": "2026-03-27T14:42:38.830329102Z"
    }
  ],
  "created_at": "2026-03-27T14:42:38.830331694Z",
  "updated_at": "2026-03-27T14:42:38.830331694Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,23 @@
 {
  "id": "9c4d9116-1d9d-4afd-a8d1-c514a678e5fa",
  "name": "call_log",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/call_log.parquet",
      "size_bytes": 35951077,
      "created_at": "2026-03-27T14:42:47.395548205Z"
    }
  ],
  "created_at": "2026-03-27T14:42:47.395555326Z",
  "updated_at": "2026-03-27T14:42:47.395555326Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -1,15 +0,0 @@
 {
  "id": "d2ce2995-9c60-49c9-9b41-197020cebaae",
  "name": "placements",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/placements.parquet",
      "size_bytes": 1213820,
      "created_at": "2026-03-27T14:00:35.885543632Z"
    }
  ],
  "created_at": "2026-03-27T14:00:35.885550623Z",
  "updated_at": "2026-03-27T14:00:35.885550623Z"
 }
@@ -0,0 +1,23 @@
 {
  "id": "d35c7941-37e2-4bde-8226-5cf69c74931a",
  "name": "placements",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/placements.parquet",
      "size_bytes": 1213820,
      "created_at": "2026-03-27T14:42:39.040983450Z"
    }
  ],
  "created_at": "2026-03-27T14:42:39.040989351Z",
  "updated_at": "2026-03-27T14:42:39.040989351Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,72 @@
 {
  "id": "d609d775-1a8e-4114-8aee-3238017229a7",
  "name": "hybrid_indexing_setup",
  "schema_fingerprint": "dfb09f4395d131e5fda5dab65f8c9dbe395d46d0e828f504e053cdb1cf23f24b",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/hybrid_indexing_setup.parquet",
      "size_bytes": 131530,
      "created_at": "2026-03-28T03:30:28.978970613Z"
    }
  ],
  "created_at": "2026-03-28T03:30:28.978971805Z",
  "updated_at": "2026-03-28T03:30:28.979244113Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "Hybrid Indexing Setup.pdf",
    "ingest_job": "ingest-1774668628978",
    "ingest_timestamp": "2026-03-28T03:30:28.978970613Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 69
 }
@@ -1,15 +0,0 @@
 {
  "id": "d8170213-d6af-4478-ae23-59f06fda3165",
  "name": "job_orders",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/job_orders.parquet",
      "size_bytes": 905534,
      "created_at": "2026-03-27T14:00:35.780022147Z"
    }
  ],
  "created_at": "2026-03-27T14:00:35.780029168Z",
  "updated_at": "2026-03-27T14:00:35.780029168Z"
 }
@@ -1,15 +0,0 @@
 {
  "id": "e26d3633-a341-4229-9819-f287d98b788a",
  "name": "candidates",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/candidates.parquet",
      "size_bytes": 10592165,
      "created_at": "2026-03-27T14:00:35.662150713Z"
    }
  ],
  "created_at": "2026-03-27T14:00:35.662162510Z",
  "updated_at": "2026-03-27T14:00:35.662162510Z"
 }
@@ -0,0 +1,100 @@
 {
  "id": "e2a8f88a-59f6-40c7-a45b-e23d8f3533b6",
  "name": "lab_trials",
  "schema_fingerprint": "1d5782349402439a7e44efd0ccab9ae64ac3044221adef9e828b60b8bbb44dd5",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/lab_trials.parquet",
      "size_bytes": 64646,
      "created_at": "2026-03-28T01:14:03.026116573Z"
    }
  ],
  "created_at": "2026-03-28T01:14:03.026117277Z",
  "updated_at": "2026-03-28T01:14:03.026247826Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "id",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "experiment_id",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "trial_num",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "config_diff",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "config_snapshot",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "scores",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "avg_score",
      "data_type": "Float64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "improved",
      "data_type": "Boolean",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "duration_ms",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "created_at",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "postgresql",
    "source_file": "127.0.0.1:5432/knowledge_base.lab_trials",
    "ingest_job": "pg-import-1774660443026",
    "ingest_timestamp": "2026-03-28T01:14:03.026116573Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 40
 }
@@ -1,15 +0,0 @@
 {
  "id": "e4b8441f-d729-4465-91fb-2ed5f481e65d",
  "name": "clients",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/clients.parquet",
      "size_bytes": 21971,
      "created_at": "2026-03-27T14:00:35.670181596Z"
    }
  ],
  "created_at": "2026-03-27T14:00:35.670184688Z",
  "updated_at": "2026-03-27T14:00:35.670184688Z"
 }
@@ -0,0 +1,240 @@
 {
  "id": "e7304f05-5278-4e17-961a-51f2588fd2aa",
  "name": "threat_intel",
  "schema_fingerprint": "df1e126046147b3de42086880e10c3501a3a615ecddf336bc24957a24c321241",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/threat_intel.parquet",
      "size_bytes": 111130,
      "created_at": "2026-03-28T01:14:03.054140697Z"
    }
  ],
  "created_at": "2026-03-28T01:14:03.054141294Z",
  "updated_at": "2026-03-28T01:14:03.054427047Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "id",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "ip",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "threat_level",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "classification",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "confidence",
      "data_type": "Float64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "summary",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "indicators",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "recommendation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "pattern",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "attack_type",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "likely_automated",
      "data_type": "Boolean",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "country",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "country_code",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "city",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "isp",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "org",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "asn",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "is_proxy",
      "data_type": "Boolean",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "is_hosting",
      "data_type": "Boolean",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "open_ports",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "blocklist_count",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "blocklist_total",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "blocklists_blocked",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "reverse_dns",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "traceroute",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "log_count",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "banned",
      "data_type": "Boolean",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "enriched_at",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "updated_at",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "raw_data",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "postgresql",
    "source_file": "127.0.0.1:5432/knowledge_base.threat_intel",
    "ingest_job": "pg-import-1774660443054",
    "ingest_timestamp": "2026-03-28T01:14:03.054140697Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 20
 }
@@ -0,0 +1,23 @@
 {
  "id": "e959ca11-9f6b-4843-864a-cc3f50a8aa60",
  "name": "email_log",
  "schema_fingerprint": "auto",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/email_log.parquet",
      "size_bytes": 16768671,
      "created_at": "2026-03-27T14:42:49.271082991Z"
    }
  ],
  "created_at": "2026-03-27T14:42:49.271091077Z",
  "updated_at": "2026-03-27T14:42:49.271091077Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [],
  "lineage": null,
  "freshness": null,
  "tags": [],
  "row_count": null
 }
@@ -0,0 +1,79 @@
 {
  "id": "f0b6f408-71a5-4365-bd1d-98c1e176096a",
  "name": "new_candidates",
  "schema_fingerprint": "0fcee1ac176b9bc45ab392bbd4401042803eb80d0de48b1fa6a5e20ffb27fa8d",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/new_candidates.parquet",
      "size_bytes": 2731,
      "created_at": "2026-03-28T01:04:22.787759218Z"
    }
  ],
  "created_at": "2026-03-28T01:04:22.787759835Z",
  "updated_at": "2026-03-28T01:04:22.788113964Z",
  "description": "",
  "owner": "",
  "sensitivity": "pii",
  "columns": [
    {
      "name": "name",
      "data_type": "Utf8",
      "sensitivity": "pii",
      "description": "",
      "is_pii": true
    },
    {
      "name": "phone",
      "data_type": "Utf8",
      "sensitivity": "pii",
      "description": "",
      "is_pii": true
    },
    {
      "name": "email",
      "data_type": "Utf8",
      "sensitivity": "pii",
      "description": "",
      "is_pii": true
    },
    {
      "name": "city",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "state",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "skills",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "years",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "new_candidates.csv",
    "ingest_job": "ingest-1774659862787",
    "ingest_timestamp": "2026-03-28T01:04:22.787759218Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 3
 }
@@ -0,0 +1,72 @@
 {
  "id": "f9055d87-c7c5-4109-b311-11fab418541f",
  "name": "spark_vs_flink_iceberg",
  "schema_fingerprint": "e3f74df7c9db965c184ff5f5ec92b0f68168a7f6e367e8253d28fca571a11688",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/spark_vs_flink_iceberg.parquet",
      "size_bytes": 163762,
      "created_at": "2026-03-28T03:35:18.667429709Z"
    }
  ],
  "created_at": "2026-03-28T03:35:18.667430594Z",
  "updated_at": "2026-03-28T03:35:18.667593640Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "Spark vs Flink Iceberg.pdf",
    "ingest_job": "ingest-1774668918667",
    "ingest_timestamp": "2026-03-28T03:35:18.667429709Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 124
 }
@@ -0,0 +1,72 @@
 {
  "id": "faef511e-4f34-4002-b2f4-8aecb8cfb80e",
  "name": "hybrid_ai_data_lake",
  "schema_fingerprint": "1fa158168cf5f542e9cdeec30c9e9aad9b12285b66e6fbdb7321e5d9f7039515",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/hybrid_ai_data_lake.parquet",
      "size_bytes": 220352,
      "created_at": "2026-03-28T03:35:16.027063136Z"
    }
  ],
  "created_at": "2026-03-28T03:35:16.027064491Z",
  "updated_at": "2026-03-28T03:35:16.027333389Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "source_file",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "conversation_title",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "turn_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "role",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "content",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page_number",
      "data_type": "Int32",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "pdf",
    "source_file": "Hybrid AI Data Lake.pdf",
    "ingest_job": "ingest-1774668916027",
    "ingest_timestamp": "2026-03-28T03:35:16.027063136Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 133
 }
@@ -0,0 +1,51 @@
 {
  "id": "fe6b6872-03f8-491a-b2c2-7d25b19cd9bb",
  "name": "knowledge_decisions",
  "schema_fingerprint": "be65fb1ba2121bcf43c195c06dbdc0ae892eb835f4a909e9c1c0438ac81a1d0d",
  "objects": [
    {
      "bucket": "data",
      "key": "datasets/knowledge_decisions.parquet",
      "size_bytes": 21274,
      "created_at": "2026-03-28T04:06:05.360578855Z"
    }
  ],
  "created_at": "2026-03-28T04:06:05.360579258Z",
  "updated_at": "2026-03-28T04:06:05.360801969Z",
  "description": "",
  "owner": "",
  "sensitivity": null,
  "columns": [
    {
      "name": "conversation",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "page",
      "data_type": "Int64",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    },
    {
      "name": "decision",
      "data_type": "Utf8",
      "sensitivity": null,
      "description": "",
      "is_pii": false
    }
  ],
  "lineage": {
    "source_system": "csv",
    "source_file": "knowledge_decisions.csv",
    "ingest_job": "ingest-1774670765360",
    "ingest_timestamp": "2026-03-28T04:06:05.360578855Z",
    "parent_datasets": []
  },
  "freshness": null,
  "tags": [],
  "row_count": 300
 }
--- a/Show More
+++ b/Show More