Observer KB UI: now shows LLM Team extracted facts + entities

- Knowledge tab: stats (categories, verified facts, entities, decisions, insights, questions), auto-generated categories, key decisions, key insights - Facts & Entities tab (NEW): 279 extracted facts from LLM Team extract mode, 264 entities as clickable chips (click → search), verified entries from secondary model - Search tab: semantic HNSW search across all knowledge - Ask AI tab: RAG with cited sources - Ingest tab: upload with progress - Entity chips link to semantic search - Category drill-down shows related content via HNSW Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
LLM Team integration: extract mode processes 8 conversations
2026-03-28 00:25:39 -05:00 · 2026-03-28 00:22:29 -05:00 · 2026-03-27 23:35:15 -05:00 · 2026-03-27 23:26:05 -05:00 · 2026-03-27 23:17:16 -05:00 · 2026-03-27 23:12:13 -05:00
151 changed files with 11400 additions and 257 deletions
@@ -8,6 +8,17 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"

+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
 [[package]]
 name = "ahash"
 version = "0.8.12"
@@ -508,6 +519,15 @@ dependencies = [
 "generic-array",
 ]

+[[package]]
+name = "block-padding"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "brotli"
 version = "8.0.2"
@@ -535,6 +555,12 @@ version = "3.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"

+[[package]]
+name = "bytecount"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -588,6 +614,15 @@ dependencies = [
 "uuid",
 ]

+[[package]]
+name = "cbc"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "cc"
 version = "1.2.58"
@@ -660,7 +695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
 dependencies = [
 "chrono",
- "phf",
+ "phf 0.12.1",
 ]

 [[package]]
@@ -690,6 +725,16 @@ dependencies = [
 "half",
 ]

+[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common",
+ "inout",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -894,6 +939,25 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.21"
@@ -2156,6 +2220,12 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
 [[package]]
 name = "fastrand"
 version = "2.3.0"
@@ -2309,8 +2379,12 @@ name = "gateway"
 version = "0.1.0"
 dependencies = [
 "aibridge",
+ "arrow",
 "axum",
 "catalogd",
+ "chrono",
+ "ingestd",
+ "journald",
 "object_store",
 "opentelemetry",
 "opentelemetry-stdout",
@@ -2327,6 +2401,7 @@ dependencies = [
 "tracing",
 "tracing-opentelemetry",
 "tracing-subscriber",
+ "vectord",
 ]

 [[package]]
@@ -2358,7 +2433,7 @@ dependencies = [
 "cfg-if",
 "js-sys",
 "libc",
- "wasi",
+ "wasi 0.11.1+wasi-snapshot-preview1",
 "wasm-bindgen",
 ]

@@ -2527,12 +2602,27 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"

+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"

+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
 [[package]]
 name = "http"
 version = "1.4.0"
@@ -2820,6 +2910,52 @@ dependencies = [
 "cfb",
 ]

+[[package]]
+name = "ingestd"
+version = "0.1.0"
+dependencies = [
+ "arrow",
+ "axum",
+ "bytes",
+ "catalogd",
+ "chrono",
+ "csv",
+ "lopdf",
+ "object_store",
+ "parquet",
+ "serde",
+ "serde_json",
+ "sha2",
+ "shared",
+ "storaged",
+ "tokio",
+ "tokio-postgres",
+ "tracing",
+]
+
+[[package]]
+name = "inout"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
+dependencies = [
+ "block-padding",
+ "generic-array",
+]
+
+[[package]]
+name = "instant-distance"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c619cdaa30bb84088963968bee12a45ea5fbbf355f2c021bcd15589f5ca494a"
+dependencies = [
+ "num_cpus",
+ "ordered-float 3.9.2",
+ "parking_lot",
+ "rand 0.8.5",
+ "rayon",
+]
+
 [[package]]
 name = "integer-encoding"
 version = "3.0.4"
@@ -2920,6 +3056,24 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "journald"
+version = "0.1.0"
+dependencies = [
+ "arrow",
+ "axum",
+ "bytes",
+ "chrono",
+ "object_store",
+ "parquet",
+ "serde",
+ "serde_json",
+ "shared",
+ "storaged",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.91"
@@ -3036,6 +3190,15 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"

+[[package]]
+name = "libredox"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -3075,6 +3238,30 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86"

+[[package]]
+name = "lopdf"
+version = "0.35.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c7c1d3350d071cb86987a6bcb205c7019a0eb70dcad92b454fec722cca8d68b"
+dependencies = [
+ "aes",
+ "cbc",
+ "chrono",
+ "encoding_rs",
+ "flate2",
+ "indexmap",
+ "itoa",
+ "log",
+ "md-5",
+ "nom",
+ "nom_locate",
+ "rangemap",
+ "rayon",
+ "thiserror 2.0.18",
+ "time",
+ "weezl",
+]
+
 [[package]]
 name = "lru-slab"
 version = "0.1.2"
@@ -3217,6 +3404,12 @@ dependencies = [
 "unicase",
 ]

+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.9"
@@ -3234,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
 dependencies = [
 "libc",
- "wasi",
+ "wasi 0.11.1+wasi-snapshot-preview1",
 "windows-sys 0.61.2",
 ]

@@ -3291,6 +3484,27 @@ dependencies = [
 "jni-sys 0.3.1",
 ]

+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "nom_locate"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3"
+dependencies = [
+ "bytecount",
+ "memchr",
+ "nom",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.50.3"
@@ -3380,6 +3594,16 @@ dependencies = [
 "libm",
 ]

+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "num_enum"
 version = "0.7.6"
@@ -3402,6 +3626,24 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "objc2-core-foundation"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "objc2-system-configuration"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396"
+dependencies = [
+ "objc2-core-foundation",
+]
+
 [[package]]
 name = "object"
 version = "0.37.3"
@@ -3518,6 +3760,15 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "ordered-float"
+version = "3.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.5"
@@ -3605,7 +3856,17 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
 dependencies = [
- "phf_shared",
+ "phf_shared 0.12.1",
+]
+
+[[package]]
+name = "phf"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
+dependencies = [
+ "phf_shared 0.13.1",
+ "serde",
 ]

 [[package]]
@@ -3617,6 +3878,15 @@ dependencies = [
 "siphasher",
 ]

+[[package]]
+name = "phf_shared"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.11"
@@ -3655,6 +3925,38 @@ version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"

+[[package]]
+name = "postgres-protocol"
+version = "0.6.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491"
+dependencies = [
+ "base64",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "hmac",
+ "md-5",
+ "memchr",
+ "rand 0.9.2",
+ "sha2",
+ "stringprep",
+]
+
+[[package]]
+name = "postgres-types"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20"
+dependencies = [
+ "bytes",
+ "chrono",
+ "fallible-iterator",
+ "postgres-protocol",
+ "serde_core",
+ "serde_json",
+]
+
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -3814,6 +4116,7 @@ dependencies = [
 "axum",
 "bytes",
 "catalogd",
+ "chrono",
 "datafusion",
 "futures",
 "object_store",
@@ -3971,12 +4274,38 @@ dependencies = [
 "getrandom 0.3.4",
 ]

+[[package]]
+name = "rangemap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
+
 [[package]]
 name = "raw-window-handle"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"

+[[package]]
+name = "rayon"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "recursive"
 version = "0.1.1"
@@ -4574,6 +4903,17 @@ dependencies = [
 "tracing",
 ]

+[[package]]
+name = "stringprep"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+ "unicode-properties",
+]
+
 [[package]]
 name = "subsecond"
 version = "0.7.3"
@@ -4709,7 +5049,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
 dependencies = [
 "byteorder",
 "integer-encoding",
- "ordered-float",
+ "ordered-float 2.10.1",
 ]

 [[package]]
@@ -4805,6 +5145,32 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "tokio-postgres"
+version = "0.7.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "futures-channel",
+ "futures-util",
+ "log",
+ "parking_lot",
+ "percent-encoding",
+ "phf 0.13.1",
+ "pin-project-lite",
+ "postgres-protocol",
+ "postgres-types",
+ "rand 0.9.2",
+ "socket2 0.6.3",
+ "tokio",
+ "tokio-util",
+ "whoami",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"
@@ -5161,12 +5527,33 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"

+[[package]]
+name = "unicode-bidi"
+version = "0.3.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"

+[[package]]
+name = "unicode-normalization"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "unicode-properties"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.13.2"
@@ -5233,6 +5620,26 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"

+[[package]]
+name = "vectord"
+version = "0.1.0"
+dependencies = [
+ "aibridge",
+ "arrow",
+ "axum",
+ "bytes",
+ "chrono",
+ "instant-distance",
+ "object_store",
+ "parquet",
+ "serde",
+ "serde_json",
+ "shared",
+ "storaged",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -5286,6 +5693,15 @@ version = "0.11.1+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"

+[[package]]
+name = "wasi"
+version = "0.14.7+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
+dependencies = [
+ "wasip2",
+]
+
 [[package]]
 name = "wasip2"
 version = "1.0.2+wasi-0.2.9"
@@ -5304,6 +5720,15 @@ dependencies = [
 "wit-bindgen",
 ]

+[[package]]
+name = "wasite"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42"
+dependencies = [
+ "wasi 0.14.7+wasi-0.2.4",
+]
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.114"
@@ -5439,6 +5864,25 @@ dependencies = [
 "rustls-pki-types",
 ]

+[[package]]
+name = "weezl"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
+
+[[package]]
+name = "whoami"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d"
+dependencies = [
+ "libc",
+ "libredox",
+ "objc2-system-configuration",
+ "wasite",
+ "web-sys",
+]
+
 [[package]]
 name = "winapi-util"
 version = "0.1.11"
@@ -7,6 +7,9 @@ members = [
    "crates/catalogd",
    "crates/queryd",
    "crates/aibridge",
+    "crates/ingestd",
+    "crates/vectord",
+    "crates/journald",
    "crates/gateway",
    "crates/ui",
 ]
@@ -38,3 +41,8 @@ opentelemetry_sdk = { version = "0.28", features = ["rt-tokio"] }
 opentelemetry-stdout = { version = "0.28", features = ["trace"] }
 tracing-opentelemetry = "0.29"
 toml = "0.8"
+csv = "1"
+lopdf = "0.35"
+encoding_rs = "0.8"
+instant-distance = "0.6"
+tokio-postgres = { version = "0.7", features = ["with-serde_json-1", "with-chrono-0_4"] }
@@ -0,0 +1,183 @@
+# PRD: Observer Knowledge Base
+
+**Status:** Active — extraction pipeline proven, needs multi-pass processing via LLM Team modes
+**Created:** 2026-03-28
+**Owner:** J
+**Branch:** `observer-kb`
+**Parent:** Lakehouse core (crates/)
+
+---
+
+## Problem
+
+AI conversations generate massive amounts of knowledge that's lost to context windows, hallucination, and scope drift. Current approach (single-pass extraction into categories) produces disorganized, hard-to-validate output. Reading raw pages is not useful — the system needs to process, clean, structure, and curate knowledge like an editorial team.
+
+## Vision
+
+A publishing pipeline that takes raw AI conversation exports and produces well-curated, validated knowledge documents. The output should be something you hand to a team — organized, filtered, and clearly presented. Questions anchor the content, with branching connections showing what was asked, what was answered, and how reasoning was directed.
+
+---
+
+## Architecture
+
+### Multi-Pass Processing via LLM Team Modes
+
+The key insight: devop.live already has 20 analysis modes that process content through structured workflows. The Observer KB should route uploaded data through these modes automatically.
+
+```
+Raw PDF Upload
+    │
+    ▼ Mode: EXTRACT
+    Structured facts, entities, relationships
+    │
+    ▼ Mode: VALIDATOR
+    Fact-check claims, score 1-10 confidence
+    │
+    ▼ Mode: HALLUCINATION
+    Verify each claim, flag likely hallucinations
+    │
+    ▼ Mode: CONSENSUS
+    Multiple passes iterate until extraction converges
+    │
+    ▼ Mode: REDTEAM
+    Find flaws in knowledge, patch them
+    │
+    ▼ Mode: PIPELINE (custom)
+    Clean → Q&A structure → topic grouping → value ranking
+    │
+    ▼ RENDER
+    Well-curated document: questions as anchors, branching connections,
+    what was asked → answered → directed
+```
+
+### Integration with devop.live (LLM Team)
+
+The Observer KB calls the LLM Team API to run modes against extracted content:
+
+| Mode | Purpose in KB |
+|---|---|
+| **extract** | Structured facts, entities, relationships from raw text |
+| **validator** | Fact-check extracted claims, confidence scores |
+| **hallucination** | Verify each factual claim independently |
+| **drift** | Process N times, find consistent vs uncertain content |
+| **redteam** | Find flaws in extracted knowledge, improve |
+| **consensus** | Iterate until multiple passes agree on correct extraction |
+| **pipeline** | Chain: clean → extract → validate → organize → render |
+| **research** | Generate follow-up questions, research in parallel |
+| **blindassembly** | Split content into parts, process independently, reassemble |
+
+### Data Flow
+
+```
+Upload Portal (Observer KB :3400)
+    │
+    ├── PDF → Lakehouse ingestd (text extraction, Parquet storage)
+    │
+    ▼
+Processing Pipeline
+    │
+    ├── Call devop.live API (:5000) with each mode
+    ├── Each mode processes content through local Ollama models
+    ├── Results stored back in Lakehouse (Parquet + catalog)
+    │
+    ▼
+Knowledge Store
+    │
+    ├── knowledge_facts: verified claims with confidence scores
+    ├── knowledge_qa: question → answer pairs with context
+    ├── knowledge_topics: auto-generated categories
+    ├── knowledge_connections: how topics relate across conversations
+    ├── knowledge_contradictions: flagged conflicts between conversations
+    ├── knowledge_validated: claims that passed hallucination check
+    │
+    ▼
+Rendering
+    │
+    ├── Structured knowledge view (web UI)
+    ├── Exportable curated document (PDF/markdown)
+    └── Question-anchored navigation (Q → A → follow-up → related)
+```
+
+---
+
+## Processing Pipeline Detail
+
+### Pass 1: Clean + Structure
+- Strip PDF artifacts (headers, footers, page numbers)
+- Identify conversation turns (user/AI)
+- Fix encoding issues (ligatures, special chars)
+- Normalize formatting
+
+### Pass 2: Q&A Extraction
+- Identify questions asked by user
+- Map each question to its answer
+- Track follow-up questions and how they branch
+- Build question→answer→follow-up chains
+
+### Pass 3: Fact Extraction + Validation
+- Run EXTRACT mode: pull structured facts, entities, relationships
+- Run VALIDATOR mode: score each fact 1-10 confidence
+- Run HALLUCINATION mode: verify claims independently
+- Flag contradictions between conversations
+
+### Pass 4: Organize + Curate
+- Run CONSENSUS mode: multiple passes converge on correct categorization
+- Auto-generate topic categories from content (not predefined)
+- Rank content by value (decisions > insights > discussion > noise)
+- Connect related content across conversations
+
+### Pass 5: Render
+- Question-anchored view: each question is a node, answers branch from it
+- Topic-grouped view: knowledge organized by auto-generated categories
+- Timeline view: how thinking evolved across conversations
+- Export as clean markdown/PDF
+
+---
+
+## Current State (as of 2026-03-28)
+
+### What's Built
+- PDF ingestion (8 PDFs, 477 pages extracted)
+- Single-pass knowledge extraction (496 topics, 300 decisions, 100 insights)
+- Semantic search via HNSW (27ms over 477 chunks)
+- RAG (question → search → LLM answer with sources)
+- 5-pass deep analysis (corpus overview, patterns, evolution, gaps, synthesis)
+- Observer KB web UI (knowledge-first, not page-first)
+- Upload portal with drag & drop
+
+### What's Not Built Yet
+- Integration with devop.live LLM Team modes
+- Multi-pass processing pipeline (extract → validate → hallucination check)
+- Q&A chain extraction (question → answer → follow-up mapping)
+- Contradiction detection between conversations
+- Curated document export (PDF/markdown)
+- Question-anchored navigation
+- Auto-processing on upload (currently manual script)
+
+### What Needs to Happen Next Session
+1. **Integrate with devop.live API** — call modes programmatically
+2. **Build the multi-pass pipeline** — extract → validate → hallucinate-check → organize
+3. **Q&A chain extraction** — questions as anchors, answers branching
+4. **Auto-process on upload** — no manual script, pipeline runs automatically
+5. **Render curated output** — readable, hand-it-to-someone quality
+
+---
+
+## Tech Stack
+
+- **Storage:** Lakehouse storaged (Parquet on object storage)
+- **Catalog:** Lakehouse catalogd (metadata, PII, lineage)
+- **Query:** Lakehouse queryd (DataFusion SQL, hot cache, paged results)
+- **Vectors:** Lakehouse vectord (HNSW search, RAG)
+- **AI Processing:** devop.live LLM Team API (20 modes, multiple models)
+- **AI Runtime:** Ollama (local models — qwen2.5, mistral, gemma2, nomic-embed-text)
+- **UI:** Pure HTML/JS on :3400
+
+---
+
+## Non-Goals (for now)
+
+- Real-time conversation ingestion (PDF exports only)
+- Multi-user collaboration
+- Cloud deployment
+- Editing extracted knowledge from UI (read-only for now)
@@ -0,0 +1,286 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Observer KB</title>
+<style>
+:root{--bg:#0a0a0f;--s1:#12121a;--s2:#1a1a28;--b:#2a2a3a;--t:#e0e0e8;--d:#888898;--a:#6c5ce7;--ok:#00d2a0;--err:#ff6b6b;--warn:#ffa94d;--f:'JetBrains Mono','Fira Code',monospace}
+*{margin:0;padding:0;box-sizing:border-box}body{font-family:var(--f);background:var(--bg);color:var(--t);font-size:13px;min-height:100vh}
+.app{max-width:1200px;margin:0 auto;padding:20px}
+header{display:flex;align-items:center;gap:20px;padding:16px 0;border-bottom:1px solid var(--b);margin-bottom:20px}
+header h1{font-size:20px;color:var(--a)}
+.tabs{display:flex;gap:2px}.tab{background:none;border:none;color:var(--d);font-family:var(--f);font-size:12px;padding:10px 16px;cursor:pointer;border-bottom:2px solid transparent}
+.tab:hover{color:var(--t)}.tab.active{color:var(--a);border-bottom-color:var(--a)}
+.hdr-stats{margin-left:auto;font-size:11px;color:var(--d)}
+.btn{background:var(--a);color:white;border:none;padding:8px 16px;border-radius:6px;font-family:var(--f);font-size:12px;cursor:pointer;font-weight:600}
+.btn:hover{opacity:.9}.btn:disabled{opacity:.5}.btn-sm{padding:5px 10px;font-size:11px}
+.hidden{display:none}
+.stats-bar{display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:8px;margin-bottom:20px}
+.stat{background:var(--s1);border:1px solid var(--b);border-radius:8px;padding:12px;text-align:center}
+.stat-val{font-size:22px;font-weight:700;color:var(--a)}.stat-label{font-size:10px;color:var(--d);text-transform:uppercase;margin-top:2px}
+.section{margin-bottom:24px}.section h2{font-size:13px;color:var(--d);text-transform:uppercase;letter-spacing:.08em;margin-bottom:10px;border-bottom:1px solid var(--b);padding-bottom:6px}
+.entry{background:var(--s1);border:1px solid var(--b);border-radius:8px;padding:14px;margin-bottom:8px}
+.badge{font-size:9px;text-transform:uppercase;letter-spacing:.08em;padding:2px 8px;border-radius:3px;display:inline-block;margin-bottom:6px;font-weight:600}
+.badge-fact{background:var(--a);color:white}.badge-entity{background:var(--ok);color:white}
+.badge-decision{background:#e67e22;color:white}.badge-insight{background:#3498db;color:white}
+.badge-question{background:var(--warn);color:#000}.badge-verified{background:var(--ok);color:white}
+.entry-text{font-size:12px;line-height:1.6}.entry-src{font-size:10px;color:var(--d);margin-top:4px}
+.cat-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(250px,1fr));gap:8px;margin-bottom:16px}
+.cat-card{background:var(--s1);border:1px solid var(--b);border-radius:8px;padding:14px;cursor:pointer;border-left:3px solid var(--a);transition:all .15s}
+.cat-card:hover{border-color:var(--a)}.cat-title{font-weight:600;font-size:13px}.cat-count{font-size:10px;color:var(--d);margin-top:2px}
+.search-row{display:flex;gap:8px;margin-bottom:16px}
+.search-row input{flex:1;background:var(--s1);border:2px solid var(--b);color:var(--t);font-family:var(--f);font-size:14px;padding:12px;border-radius:8px;outline:none}
+.search-row input:focus{border-color:var(--a)}
+.ask-input{width:100%;background:var(--s1);border:2px solid var(--b);color:var(--t);font-family:var(--f);font-size:13px;padding:12px;border-radius:8px;resize:vertical;min-height:80px;outline:none;margin-bottom:8px}
+.ask-input:focus{border-color:var(--a)}
+.ai-answer{background:var(--s2);border:1px solid var(--b);border-radius:8px;padding:16px;margin-top:12px;line-height:1.7;white-space:pre-wrap}
+.ai-meta{font-size:10px;color:var(--d);margin-top:6px}
+.upload-zone{border:2px dashed var(--b);border-radius:12px;padding:36px;text-align:center;cursor:pointer;margin-bottom:16px}
+.upload-zone:hover{border-color:var(--a);background:rgba(108,92,231,.03)}
+.upload-zone h3{margin-bottom:4px}.upload-zone p{color:var(--d);font-size:11px}
+.file-item{display:flex;align-items:center;justify-content:space-between;padding:8px 12px;border-bottom:1px solid var(--b);font-size:12px}
+.done{color:var(--ok)}.failed{color:var(--err)}.uploading{color:var(--a)}
+.result{background:var(--s2);border:1px solid var(--b);border-radius:8px;padding:12px;margin-bottom:6px;border-left:3px solid var(--a)}
+.result-score{font-size:10px;color:var(--a);margin-bottom:2px}.result-text{font-size:12px;line-height:1.5}
+.empty{color:var(--d);text-align:center;padding:32px}
+</style>
+</head>
+<body>
+<div class="app">
+<header>
+<h1>OBSERVER KB</h1>
+<div class="tabs">
+<button class="tab active" onclick="showTab('knowledge')">Knowledge</button>
+<button class="tab" onclick="showTab('facts')">Facts &amp; Entities</button>
+<button class="tab" onclick="showTab('explore')">Search</button>
+<button class="tab" onclick="showTab('ask')">Ask AI</button>
+<button class="tab" onclick="showTab('upload')">Ingest</button>
+</div>
+<div class="hdr-stats" id="hdr-stats"></div>
+</header>
+
+<div id="tab-knowledge"></div>
+<div id="tab-facts" class="hidden"></div>
+<div id="tab-explore" class="hidden"></div>
+<div id="tab-ask" class="hidden"></div>
+<div id="tab-upload" class="hidden"></div>
+</div>
+
+<script>
+const API=`http://${location.hostname}:3100`;
+const $=id=>document.getElementById(id);
+async function q(sql){return(await(await fetch(API+'/query/sql',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({sql})})).json())}
+async function searchHNSW(query,k=10){try{return(await(await fetch(API+'/vectors/hnsw/search',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({index_name:'observer_kb',query,top_k:k})})).json()).results||[]}catch{return[]}}
+
+function showTab(name){
+['knowledge','facts','explore','ask','upload'].forEach((t,i)=>{
+$('tab-'+t).classList.toggle('hidden',t!==name);
+document.querySelectorAll('.tab')[i].classList.toggle('active',t===name);
+});
+({knowledge:loadKnowledge,facts:loadFacts,explore:loadExplore,ask:loadAsk,upload:loadUpload})[name]();
+}
+
+function makeEntry(type,text,source){
+const e=document.createElement('div');e.className='entry';
+const b=document.createElement('span');b.className='badge badge-'+type;b.textContent=type;
+const t=document.createElement('div');t.className='entry-text';t.textContent=text;
+const s=document.createElement('div');s.className='entry-src';s.textContent=source;
+e.append(b,t,s);return e;
+}
+
+// === KNOWLEDGE (categories + decisions + insights) ===
+async function loadKnowledge(){
+const el=$('tab-knowledge');el.textContent='';
+try{
+const[topics,decisions,insights,questions,extractions]=await Promise.all([
+q("SELECT topic,COUNT(*) as n,COUNT(DISTINCT conversation) as c FROM knowledge_topics GROUP BY topic ORDER BY n DESC LIMIT 15"),
+q("SELECT conversation,page,decision FROM knowledge_decisions LIMIT 12"),
+q("SELECT conversation,page,insight FROM knowledge_insights LIMIT 8"),
+q("SELECT COUNT(*) as n FROM knowledge_questions"),
+q("SELECT type,COUNT(*) as n FROM processed_extractions GROUP BY type"),
+]);
+
+const factCount=extractions.rows.find(r=>r.type==='fact')?.n||0;
+const entityCount=extractions.rows.find(r=>r.type==='entity')?.n||0;
+
+// Stats
+const sb=document.createElement('div');sb.className='stats-bar';
+[{v:topics.row_count,l:'Categories'},{v:factCount,l:'Verified Facts'},{v:entityCount,l:'Entities'},{v:decisions.row_count,l:'Decisions'},{v:insights.row_count,l:'Insights'},{v:questions.rows[0]?.n||0,l:'Open Questions'}].forEach(s=>{
+const d=document.createElement('div');d.className='stat';
+const v=document.createElement('div');v.className='stat-val';v.textContent=s.v;
+const l=document.createElement('div');l.className='stat-label';l.textContent=s.l;
+d.append(v,l);sb.appendChild(d);
+});
+el.appendChild(sb);
+$('hdr-stats').textContent=factCount+' facts · '+entityCount+' entities · '+decisions.row_count+' decisions';
+
+// Categories
+const cs=document.createElement('div');cs.className='section';
+cs.insertAdjacentHTML('beforeend','<h2>Auto-Generated Categories</h2>');
+const cg=document.createElement('div');cg.className='cat-grid';
+topics.rows.forEach(t=>{
+const c=document.createElement('div');c.className='cat-card';
+const n=document.createElement('div');n.className='cat-title';n.textContent=t.topic;
+const m=document.createElement('div');m.className='cat-count';m.textContent=t.n+' mentions · '+t.c+' conversations';
+c.append(n,m);c.onclick=()=>drillTopic(t.topic);cg.appendChild(c);
+});
+cs.appendChild(cg);el.appendChild(cs);
+
+// Decisions
+const ds=document.createElement('div');ds.className='section';
+ds.insertAdjacentHTML('beforeend','<h2>Key Decisions</h2>');
+decisions.rows.forEach(d=>ds.appendChild(makeEntry('decision',d.decision,d.conversation+' p'+d.page)));
+el.appendChild(ds);
+
+// Insights
+const is=document.createElement('div');is.className='section';
+is.insertAdjacentHTML('beforeend','<h2>Key Insights</h2>');
+insights.rows.forEach(d=>is.appendChild(makeEntry('insight',d.insight,d.conversation+' p'+d.page)));
+el.appendChild(is);
+}catch(e){el.textContent='Error: '+e}
+}
+
+async function drillTopic(topic){
+const el=$('tab-knowledge');el.textContent='';
+const back=document.createElement('button');back.className='btn btn-sm';back.textContent='← Back';back.onclick=loadKnowledge;
+const h=document.createElement('h2');h.style.cssText='font-size:18px;margin:12px 0';h.textContent=topic;
+el.append(back,h);
+const results=await searchHNSW(topic,10);
+const s=document.createElement('div');s.className='section';
+s.insertAdjacentHTML('beforeend','<h2>Related Content</h2>');
+results.forEach(r=>{
+const e=document.createElement('div');e.className='result';
+const sc=document.createElement('div');sc.className='result-score';sc.textContent='['+r.score.toFixed(3)+'] '+r.doc_id;
+const t=document.createElement('div');t.className='result-text';t.textContent=r.chunk_text.substring(0,300);
+e.append(sc,t);s.appendChild(e);
+});
+el.appendChild(s);
+}
+
+// === FACTS & ENTITIES (from LLM Team extract mode) ===
+async function loadFacts(){
+const el=$('tab-facts');el.textContent='';
+try{
+const[facts,entities,verified]=await Promise.all([
+q("SELECT conversation,page,content FROM processed_extractions WHERE type='fact' ORDER BY conversation,page LIMIT 30"),
+q("SELECT conversation,content,COUNT(*) as n FROM processed_extractions WHERE type='entity' GROUP BY conversation,content ORDER BY n DESC LIMIT 30"),
+q("SELECT conversation,page,content FROM processed_validated LIMIT 10"),
+]);
+
+const fs=document.createElement('div');fs.className='section';
+fs.insertAdjacentHTML('beforeend','<h2>Extracted Facts ('+facts.row_count+' total — via LLM Team Extract Mode)</h2>');
+facts.rows.forEach(r=>fs.appendChild(makeEntry('fact',r.content,r.conversation+' p'+r.page)));
+el.appendChild(fs);
+
+const es=document.createElement('div');es.className='section';
+es.insertAdjacentHTML('beforeend','<h2>Entities ('+entities.row_count+' unique)</h2>');
+const eg=document.createElement('div');eg.style.cssText='display:flex;flex-wrap:wrap;gap:6px';
+entities.rows.forEach(r=>{
+const chip=document.createElement('span');
+chip.style.cssText='background:var(--s2);border:1px solid var(--b);padding:4px 10px;border-radius:4px;font-size:11px;cursor:pointer';
+chip.textContent=r.content+(r.n>1?' ('+r.n+')':'');
+chip.onclick=()=>{showTab('explore');setTimeout(()=>{document.getElementById('searchInput').value=r.content;doSearch()},100)};
+eg.appendChild(chip);
+});
+es.appendChild(eg);el.appendChild(es);
+
+if(verified.rows.length){
+const vs=document.createElement('div');vs.className='section';
+vs.insertAdjacentHTML('beforeend','<h2>Verified by Secondary Model</h2>');
+verified.rows.forEach(r=>vs.appendChild(makeEntry('verified',r.content,r.conversation+' p'+r.page)));
+el.appendChild(vs);
+}
+}catch(e){el.textContent='Error: '+e}
+}
+
+// === SEARCH ===
+function loadExplore(){
+const el=$('tab-explore');el.textContent='';
+const row=document.createElement('div');row.className='search-row';
+const input=document.createElement('input');input.id='searchInput';input.placeholder='Search by meaning across all knowledge...';
+const btn=document.createElement('button');btn.className='btn';btn.textContent='Search';
+btn.onclick=doSearch;input.onkeydown=e=>{if(e.key==='Enter')doSearch()};
+row.append(input,btn);
+const res=document.createElement('div');res.id='searchResults';
+const hint=document.createElement('div');hint.className='empty';hint.textContent='Try: "object storage architecture", "data ingestion trade-offs", "CRM options"';
+el.append(row,res,hint);
+}
+async function doSearch(){
+const query=document.getElementById('searchInput').value;if(!query)return;
+const el=$('searchResults');el.textContent='searching...';
+const results=await searchHNSW(query,15);
+el.textContent='';
+if(!results.length){el.textContent='No results';return}
+results.forEach(r=>{
+const e=document.createElement('div');e.className='result';
+const s=document.createElement('div');s.className='result-score';s.textContent='['+r.score.toFixed(3)+'] '+r.doc_id.replace(/_/g,' ');
+const t=document.createElement('div');t.className='result-text';t.textContent=r.chunk_text.substring(0,400);
+e.append(s,t);el.appendChild(e);
+});
+}
+
+// === ASK AI ===
+function loadAsk(){
+const el=$('tab-ask');el.textContent='';
+const ta=document.createElement('textarea');ta.className='ask-input';ta.id='askInput';
+ta.placeholder='Ask a question about your knowledge base...\n\nExamples:\n- What architecture decisions were made about Iceberg?\n- Compare Spark vs Flink approaches\n- What approaches were abandoned and why?';
+const btn=document.createElement('button');btn.className='btn';btn.id='askBtn';btn.textContent='Ask AI';
+btn.onclick=doAsk;
+const resp=document.createElement('div');resp.id='askResp';
+el.append(ta,btn,resp);
+}
+async function doAsk(){
+const question=$('askInput').value;if(!question)return;
+$('askBtn').disabled=true;$('askBtn').textContent='thinking...';
+const el=$('askResp');el.textContent='searching knowledge...';
+try{
+const r=await(await fetch(API+'/vectors/rag',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({index_name:'observer_kb',question,top_k:5})})).json();
+el.textContent='';
+const ans=document.createElement('div');ans.className='ai-answer';ans.textContent=r.answer;
+const meta=document.createElement('div');meta.className='ai-meta';meta.textContent='Model: '+r.model+(r.tokens_generated?' · '+r.tokens_generated+' tokens':'');
+el.append(ans,meta);
+if(r.sources?.length){
+const sh=document.createElement('h2');sh.style.cssText='font-size:11px;color:var(--d);margin-top:14px';sh.textContent='SOURCES';el.appendChild(sh);
+r.sources.forEach(s=>{
+const e=document.createElement('div');e.className='result';e.style.marginTop='4px';
+const sc=document.createElement('div');sc.className='result-score';sc.textContent='['+s.score.toFixed(3)+'] '+s.doc_id;
+const t=document.createElement('div');t.className='result-text';t.style.fontSize='11px';t.textContent=s.chunk_text.substring(0,200);
+e.append(sc,t);el.appendChild(e);
+});
+}
+}catch(e){el.textContent='Error: '+e}
+$('askBtn').disabled=false;$('askBtn').textContent='Ask AI';
+}
+
+// === UPLOAD ===
+function loadUpload(){
+const el=$('tab-upload');el.textContent='';
+const zone=document.createElement('div');zone.className='upload-zone';
+zone.onclick=()=>{const i=document.createElement('input');i.type='file';i.multiple=true;i.accept='.pdf,.txt,.csv,.json';i.onchange=e=>[...e.target.files].forEach(uploadFile);i.click()};
+zone.ondragover=e=>{e.preventDefault();zone.style.borderColor='var(--a)'};
+zone.ondragleave=()=>zone.style.borderColor='';
+zone.ondrop=e=>{e.preventDefault();zone.style.borderColor='';[...e.dataTransfer.files].forEach(uploadFile)};
+const h=document.createElement('h3');h.textContent='Drop files to ingest';
+const p=document.createElement('p');p.textContent='PDF, CSV, JSON, Text — auto-extracted and indexed';
+zone.append(h,p);
+const list=document.createElement('div');list.id='upList';
+el.append(zone,list);
+}
+function uploadFile(file){
+const item=document.createElement('div');item.className='file-item';
+const name=document.createElement('span');name.textContent=file.name;
+const status=document.createElement('span');status.className='uploading';status.textContent='uploading...';
+item.append(name,status);$('upList').prepend(item);
+const fd=new FormData();fd.append('file',file);
+const xhr=new XMLHttpRequest();xhr.open('POST',API+'/ingest/file');
+xhr.upload.onprogress=e=>{if(e.lengthComputable)status.textContent=Math.round(e.loaded/e.total*100)+'%'};
+xhr.onload=()=>{status.className=xhr.status<300?'done':'failed';status.textContent=xhr.status<300?'ingested':'failed'};
+xhr.onerror=()=>{status.className='failed';status.textContent='error'};
+xhr.send(fd);
+}
+
+loadKnowledge();
+</script>
+</body>
+</html>
@@ -1,4 +1,7 @@
-use shared::types::{DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint};
+use shared::types::{
+    DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint,
+    ColumnMeta, Lineage, FreshnessContract, Sensitivity,
+};
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
@@ -6,6 +9,19 @@ use tokio::sync::RwLock;
 use storaged::ops;
 use object_store::ObjectStore;

+/// Partial metadata update — only set fields are applied.
+#[derive(Debug, Clone, Default, serde::Deserialize)]
+pub struct MetadataUpdate {
+    pub description: Option<String>,
+    pub owner: Option<String>,
+    pub sensitivity: Option<Sensitivity>,
+    pub tags: Option<Vec<String>>,
+    pub columns: Option<Vec<ColumnMeta>>,
+    pub lineage: Option<Lineage>,
+    pub freshness: Option<FreshnessContract>,
+    pub row_count: Option<u64>,
+}
+
 const MANIFEST_PREFIX: &str = "_catalog/manifests";

 /// In-memory dataset registry backed by manifest persistence in object storage.
@@ -54,6 +70,14 @@ impl Registry {
            objects,
            created_at: now,
            updated_at: now,
+            description: String::new(),
+            owner: String::new(),
+            sensitivity: None,
+            columns: vec![],
+            lineage: None,
+            freshness: None,
+            tags: vec![],
+            row_count: None,
        };

        // Write-ahead: persist before in-memory update
@@ -68,6 +92,36 @@ impl Registry {
        Ok(manifest)
    }

+    /// Update metadata on an existing dataset (owner, description, tags, sensitivity, etc.)
+    pub async fn update_metadata(
+        &self,
+        name: &str,
+        updates: MetadataUpdate,
+    ) -> Result<DatasetManifest, String> {
+        let mut datasets = self.datasets.write().await;
+        let manifest = datasets.values_mut()
+            .find(|d| d.name == name)
+            .ok_or_else(|| format!("dataset not found: {name}"))?;
+
+        if let Some(desc) = updates.description { manifest.description = desc; }
+        if let Some(owner) = updates.owner { manifest.owner = owner; }
+        if let Some(sens) = updates.sensitivity { manifest.sensitivity = Some(sens); }
+        if let Some(tags) = updates.tags { manifest.tags = tags; }
+        if let Some(cols) = updates.columns { manifest.columns = cols; }
+        if let Some(lineage) = updates.lineage { manifest.lineage = Some(lineage); }
+        if let Some(freshness) = updates.freshness { manifest.freshness = Some(freshness); }
+        if let Some(count) = updates.row_count { manifest.row_count = Some(count); }
+        manifest.updated_at = chrono::Utc::now();
+
+        // Persist
+        let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
+        let json = serde_json::to_vec_pretty(manifest).map_err(|e| e.to_string())?;
+        ops::put(&self.store, &manifest_key, json.into()).await?;
+
+        let result = manifest.clone();
+        Ok(result)
+    }
+
    /// Get a dataset by ID.
    pub async fn get(&self, id: &DatasetId) -> Option<DatasetManifest> {
        let datasets = self.datasets.read().await;
@@ -18,6 +18,7 @@ pub fn router(registry: Registry) -> Router {
        .route("/datasets", get(list_datasets))
        .route("/datasets/{id}", get(get_dataset))
        .route("/datasets/by-name/{name}", get(get_dataset_by_name))
+        .route("/datasets/by-name/{name}/metadata", post(update_metadata))
        .with_state(registry)
 }

@@ -47,6 +48,15 @@ struct DatasetResponse {
    objects: Vec<ObjectRefResponse>,
    created_at: String,
    updated_at: String,
+    // Rich metadata
+    description: String,
+    owner: String,
+    sensitivity: Option<shared::types::Sensitivity>,
+    columns: Vec<shared::types::ColumnMeta>,
+    lineage: Option<shared::types::Lineage>,
+    freshness: Option<shared::types::FreshnessContract>,
+    tags: Vec<String>,
+    row_count: Option<u64>,
 }

 #[derive(Serialize)]
@@ -71,6 +81,14 @@ impl From<&shared::types::DatasetManifest> for DatasetResponse {
            }).collect(),
            created_at: m.created_at.to_rfc3339(),
            updated_at: m.updated_at.to_rfc3339(),
+            description: m.description.clone(),
+            owner: m.owner.clone(),
+            sensitivity: m.sensitivity.clone(),
+            columns: m.columns.clone(),
+            lineage: m.lineage.clone(),
+            freshness: m.freshness.clone(),
+            tags: m.tags.clone(),
+            row_count: m.row_count,
        }
    }
 }
@@ -123,3 +141,14 @@ async fn get_dataset_by_name(
        None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {name}"))),
    }
 }
+
+async fn update_metadata(
+    State(registry): State<Registry>,
+    Path(name): Path<String>,
+    Json(updates): Json<crate::registry::MetadataUpdate>,
+) -> impl IntoResponse {
+    match registry.update_metadata(&name, updates).await {
+        Ok(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
@@ -9,6 +9,9 @@ storaged = { path = "../storaged" }
 catalogd = { path = "../catalogd" }
 queryd = { path = "../queryd" }
 aibridge = { path = "../aibridge" }
+ingestd = { path = "../ingestd" }
+vectord = { path = "../vectord" }
+journald = { path = "../journald" }
 tokio = { workspace = true }
 axum = { workspace = true }
 serde = { workspace = true }
@@ -23,3 +26,5 @@ opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 opentelemetry-stdout = { workspace = true }
 tracing-opentelemetry = { workspace = true }
+arrow = { workspace = true }
+chrono = { workspace = true }
@@ -0,0 +1,155 @@
+/// Access control: field-level sensitivity enforcement, column masking, query audit.
+/// Evaluates policies at query time — not at storage level.
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use shared::types::Sensitivity;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+/// An agent's role determines what sensitivity levels they can see.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AgentRole {
+    pub agent_name: String,
+    pub role: String,                        // "recruiter", "admin", "analyst", "agent"
+    pub allowed_sensitivity: Vec<Sensitivity>, // what they can see unmasked
+}
+
+/// A query audit entry.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct QueryAudit {
+    pub id: String,
+    pub agent: String,
+    pub sql: String,
+    pub datasets_accessed: Vec<String>,
+    pub pii_fields_accessed: Vec<String>,
+    pub timestamp: DateTime<Utc>,
+    pub row_count: usize,
+    pub allowed: bool,
+    pub masked_fields: Vec<String>,
+}
+
+/// Access control manager.
+#[derive(Clone)]
+pub struct AccessControl {
+    roles: Arc<RwLock<HashMap<String, AgentRole>>>,
+    audit_log: Arc<RwLock<Vec<QueryAudit>>>,
+    enabled: bool,
+}
+
+impl AccessControl {
+    pub fn new(enabled: bool) -> Self {
+        let ac = Self {
+            roles: Arc::new(RwLock::new(HashMap::new())),
+            audit_log: Arc::new(RwLock::new(Vec::new())),
+            enabled,
+        };
+        ac
+    }
+
+    /// Register default roles.
+    pub async fn register_defaults(&self) {
+        let defaults = vec![
+            AgentRole {
+                agent_name: "admin".into(),
+                role: "admin".into(),
+                allowed_sensitivity: vec![
+                    Sensitivity::Public, Sensitivity::Internal,
+                    Sensitivity::Pii, Sensitivity::Phi, Sensitivity::Financial,
+                ],
+            },
+            AgentRole {
+                agent_name: "recruiter".into(),
+                role: "recruiter".into(),
+                allowed_sensitivity: vec![
+                    Sensitivity::Public, Sensitivity::Internal, Sensitivity::Pii,
+                ],
+            },
+            AgentRole {
+                agent_name: "analyst".into(),
+                role: "analyst".into(),
+                allowed_sensitivity: vec![
+                    Sensitivity::Public, Sensitivity::Internal, Sensitivity::Financial,
+                ],
+            },
+            AgentRole {
+                agent_name: "agent".into(),
+                role: "agent".into(),
+                allowed_sensitivity: vec![
+                    Sensitivity::Public, Sensitivity::Internal,
+                ],
+            },
+        ];
+
+        let mut roles = self.roles.write().await;
+        for role in defaults {
+            roles.insert(role.agent_name.clone(), role);
+        }
+    }
+
+    /// Register or update an agent role.
+    pub async fn set_role(&self, role: AgentRole) {
+        self.roles.write().await.insert(role.agent_name.clone(), role);
+    }
+
+    /// Get an agent's role.
+    pub async fn get_role(&self, agent: &str) -> Option<AgentRole> {
+        self.roles.read().await.get(agent).cloned()
+    }
+
+    /// List all roles.
+    pub async fn list_roles(&self) -> Vec<AgentRole> {
+        self.roles.read().await.values().cloned().collect()
+    }
+
+    /// Check if an agent can see a field with given sensitivity.
+    pub async fn can_access(&self, agent: &str, sensitivity: &Sensitivity) -> bool {
+        if !self.enabled { return true; }
+        match self.roles.read().await.get(agent) {
+            Some(role) => role.allowed_sensitivity.contains(sensitivity),
+            None => false, // unknown agent = no access
+        }
+    }
+
+    /// Determine which fields should be masked for an agent.
+    pub async fn masked_fields(
+        &self,
+        agent: &str,
+        columns: &[shared::types::ColumnMeta],
+    ) -> Vec<String> {
+        if !self.enabled { return vec![]; }
+
+        let role = match self.roles.read().await.get(agent) {
+            Some(r) => r.clone(),
+            None => return columns.iter().filter(|c| c.sensitivity.is_some()).map(|c| c.name.clone()).collect(),
+        };
+
+        columns.iter()
+            .filter(|col| {
+                if let Some(ref sens) = col.sensitivity {
+                    !role.allowed_sensitivity.contains(sens)
+                } else {
+                    false
+                }
+            })
+            .map(|col| col.name.clone())
+            .collect()
+    }
+
+    /// Log a query for audit.
+    pub async fn log_query(&self, audit: QueryAudit) {
+        self.audit_log.write().await.push(audit);
+    }
+
+    /// Get recent audit entries.
+    pub async fn recent_audit(&self, limit: usize) -> Vec<QueryAudit> {
+        let log = self.audit_log.read().await;
+        let start = log.len().saturating_sub(limit);
+        log[start..].iter().rev().cloned().collect()
+    }
+
+    pub fn is_enabled(&self) -> bool {
+        self.enabled
+    }
+}
@@ -0,0 +1,62 @@
+use axum::{
+    Json, Router,
+    extract::{Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use serde::Deserialize;
+
+use crate::access::{AccessControl, AgentRole};
+
+pub fn router(ac: AccessControl) -> Router {
+    Router::new()
+        .route("/roles", get(list_roles))
+        .route("/roles", post(set_role))
+        .route("/audit", get(query_audit))
+        .route("/check", post(check_access))
+        .with_state(ac)
+}
+
+async fn list_roles(State(ac): State<AccessControl>) -> impl IntoResponse {
+    Json(ac.list_roles().await)
+}
+
+async fn set_role(
+    State(ac): State<AccessControl>,
+    Json(role): Json<AgentRole>,
+) -> impl IntoResponse {
+    let name = role.agent_name.clone();
+    ac.set_role(role).await;
+    (StatusCode::OK, format!("role set: {name}"))
+}
+
+#[derive(Deserialize)]
+struct AuditQuery {
+    limit: Option<usize>,
+}
+
+async fn query_audit(
+    State(ac): State<AccessControl>,
+    Query(q): Query<AuditQuery>,
+) -> impl IntoResponse {
+    Json(ac.recent_audit(q.limit.unwrap_or(50)).await)
+}
+
+#[derive(Deserialize)]
+struct CheckRequest {
+    agent: String,
+    sensitivity: shared::types::Sensitivity,
+}
+
+async fn check_access(
+    State(ac): State<AccessControl>,
+    Json(req): Json<CheckRequest>,
+) -> impl IntoResponse {
+    let allowed = ac.can_access(&req.agent, &req.sensitivity).await;
+    Json(serde_json::json!({
+        "agent": req.agent,
+        "sensitivity": req.sensitivity,
+        "allowed": allowed,
+    }))
+}
@@ -1,7 +1,10 @@
+mod access;
+mod access_service;
 mod auth;
 mod observability;
+mod tools;

-use axum::{Router, routing::get};
+use axum::{Router, extract::DefaultBodyLimit, routing::get};
 use proto::lakehouse::catalog_service_server::CatalogServiceServer;
 use shared::config::Config;
 use tower_http::cors::{Any, CorsLayer};
@@ -30,8 +33,22 @@ async fn main() {
        tracing::warn!("catalog rebuild failed (empty store?): {e}");
    }

-    // Query engine
-    let engine = queryd::context::QueryEngine::new(registry.clone(), store.clone());
+    // Query engine with 16GB memory cache
+    let cache = queryd::cache::MemCache::new(16 * 1024 * 1024 * 1024);
+    let engine = queryd::context::QueryEngine::new(registry.clone(), store.clone(), cache);
+
+    // Event journal — append-only mutation log (flush every 100 events)
+    let journal = journald::journal::Journal::new(store.clone(), 100);
+
+    // Access control
+    let access = access::AccessControl::new(config.auth.enabled);
+    access.register_defaults().await;
+
+    // Workspace manager for agent-specific overlays
+    let workspace_mgr = queryd::workspace::WorkspaceManager::new(store.clone());
+    if let Err(e) = workspace_mgr.rebuild().await {
+        tracing::warn!("workspace rebuild: {e}");
+    }

    // AI sidecar client
    let ai_client = aibridge::client::AiClient::new(&config.sidecar.url);
@@ -39,10 +56,36 @@ async fn main() {
    // HTTP router
    let mut app = Router::new()
        .route("/health", get(health))
-        .nest("/storage", storaged::service::router(store))
+        .nest("/storage", storaged::service::router(store.clone()))
        .nest("/catalog", catalogd::service::router(registry.clone()))
-        .nest("/query", queryd::service::router(engine))
-        .nest("/ai", aibridge::service::router(ai_client));
+        .nest("/query", queryd::service::router(engine.clone()))
+        .nest("/ai", aibridge::service::router(ai_client.clone()))
+        .nest("/ingest", ingestd::service::router(ingestd::service::IngestState {
+            store: store.clone(),
+            registry: registry.clone(),
+        }))
+        .nest("/vectors", vectord::service::router({
+            let index_reg = vectord::index_registry::IndexRegistry::new(store.clone());
+            let _ = index_reg.rebuild().await;
+            vectord::service::VectorState {
+                store: store.clone(),
+                ai_client: ai_client.clone(),
+                job_tracker: vectord::jobs::JobTracker::new(),
+                index_registry: index_reg,
+                hnsw_store: vectord::hnsw::HnswStore::new(),
+            }
+        }))
+        .nest("/workspaces", queryd::workspace_service::router(workspace_mgr))
+        .nest("/journal", journald::service::router(journal))
+        .nest("/access", access_service::router(access))
+        .nest("/tools", tools::service::router({
+            let tool_reg = tools::registry::ToolRegistry::new_with_defaults();
+            tool_reg.register_defaults().await;
+            tools::ToolState {
+                registry: tool_reg,
+                query_fn: tools::QueryExecutor::new(engine.clone()),
+            }
+        }));

    // Auth middleware (if enabled)
    if config.auth.enabled {
@@ -58,12 +101,20 @@ async fn main() {
    }

    app = app
+        .layer(DefaultBodyLimit::max(256 * 1024 * 1024)) // 256MB
        .layer(CorsLayer::new()
            .allow_origin(Any)
            .allow_methods(Any)
            .allow_headers(Any))
        .layer(TraceLayer::new_for_http());

+    // File watcher — auto-ingest from ./inbox
+    let _watcher = ingestd::watcher::spawn_watcher(
+        ingestd::watcher::WatchConfig::default(),
+        store.clone(),
+        registry.clone(),
+    );
+
    // Start gRPC server on port+1
    let grpc_port = config.gateway.port + 1;
    let catalog_grpc = catalogd::grpc::CatalogGrpc::new(registry);
@@ -0,0 +1,47 @@
+pub mod registry;
+pub mod service;
+
+use queryd::context::QueryEngine;
+use arrow::json::writer::{JsonArray, Writer as JsonWriter};
+
+/// State for the tool system.
+#[derive(Clone)]
+pub struct ToolState {
+    pub registry: registry::ToolRegistry,
+    pub query_fn: QueryExecutor,
+}
+
+/// Wraps QueryEngine to provide a simple execute interface for tools.
+#[derive(Clone)]
+pub struct QueryExecutor {
+    engine: QueryEngine,
+}
+
+impl QueryExecutor {
+    pub fn new(engine: QueryEngine) -> Self {
+        Self { engine }
+    }
+
+    /// Execute SQL and return (rows as JSON, row count).
+    pub async fn execute(&self, sql: &str) -> Result<(serde_json::Value, usize), String> {
+        let batches = self.engine.query(sql).await?;
+
+        if batches.is_empty() {
+            return Ok((serde_json::Value::Array(vec![]), 0));
+        }
+
+        let mut buf = Vec::new();
+        let mut writer = JsonWriter::<_, JsonArray>::new(&mut buf);
+        for batch in &batches {
+            writer.write(batch).map_err(|e| format!("JSON write: {e}"))?;
+        }
+        writer.finish().map_err(|e| format!("JSON finish: {e}"))?;
+        drop(writer);
+
+        let rows: serde_json::Value = serde_json::from_slice(&buf)
+            .map_err(|e| format!("JSON parse: {e}"))?;
+        let count = rows.as_array().map(|a| a.len()).unwrap_or(0);
+
+        Ok((rows, count))
+    }
+}
@@ -0,0 +1,274 @@
+/// Tool Registry: named, governed business actions for AI agents.
+/// Instead of raw SQL, agents call validated tools with audit trails.
+///
+/// Each tool has:
+/// - Name and description (for LLM tool-use)
+/// - Parameter schema (validated before execution)
+/// - Permission level (read / write / admin)
+/// - Audit logging (every invocation recorded)
+/// - Rate limiting (per agent)
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+/// Permission level for a tool.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Permission {
+    Read,   // auto-approved, no side effects
+    Write,  // modifies data, logged
+    Admin,  // destructive, requires confirmation
+}
+
+/// Parameter definition for a tool.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ParamDef {
+    pub name: String,
+    pub param_type: String, // "string", "integer", "boolean", "float"
+    pub required: bool,
+    pub description: String,
+    pub default: Option<serde_json::Value>,
+}
+
+/// Tool definition — what agents see and can call.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ToolDef {
+    pub name: String,
+    pub description: String,
+    pub permission: Permission,
+    pub parameters: Vec<ParamDef>,
+    pub returns: String,          // description of return value
+    pub sql_template: String,     // SQL with {param} placeholders
+    pub category: String,         // "candidates", "placements", "analytics"
+}
+
+/// Audit log entry for a tool invocation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ToolInvocation {
+    pub id: String,
+    pub tool_name: String,
+    pub agent: String,
+    pub params: serde_json::Value,
+    pub permission: Permission,
+    pub timestamp: DateTime<Utc>,
+    pub success: bool,
+    pub error: Option<String>,
+    pub rows_returned: Option<usize>,
+}
+
+/// The registry — holds tool definitions and audit log.
+#[derive(Clone)]
+pub struct ToolRegistry {
+    tools: Arc<RwLock<HashMap<String, ToolDef>>>,
+    audit_log: Arc<RwLock<Vec<ToolInvocation>>>,
+}
+
+impl ToolRegistry {
+    pub fn new() -> Self {
+        let registry = Self {
+            tools: Arc::new(RwLock::new(HashMap::new())),
+            audit_log: Arc::new(RwLock::new(Vec::new())),
+        };
+        // Register built-in staffing tools
+        tokio::task::block_in_place(|| {
+            tokio::runtime::Handle::current().block_on(registry.register_defaults())
+        });
+        registry
+    }
+
+    pub fn new_with_defaults() -> Self {
+        let registry = Self {
+            tools: Arc::new(RwLock::new(HashMap::new())),
+            audit_log: Arc::new(RwLock::new(Vec::new())),
+        };
+        registry
+    }
+
+    /// Register default staffing tools.
+    pub async fn register_defaults(&self) {
+        let tools = vec![
+            ToolDef {
+                name: "search_candidates".into(),
+                description: "Search for candidates by skills, city, state, availability, and experience. Returns matching candidates with contact info.".into(),
+                permission: Permission::Read,
+                parameters: vec![
+                    ParamDef { name: "skills".into(), param_type: "string".into(), required: false, description: "Comma-separated skills to match (uses LIKE)".into(), default: None },
+                    ParamDef { name: "city".into(), param_type: "string".into(), required: false, description: "City name".into(), default: None },
+                    ParamDef { name: "state".into(), param_type: "string".into(), required: false, description: "State abbreviation".into(), default: None },
+                    ParamDef { name: "min_years".into(), param_type: "integer".into(), required: false, description: "Minimum years of experience".into(), default: Some(serde_json::json!(0)) },
+                    ParamDef { name: "status".into(), param_type: "string".into(), required: false, description: "Candidate status (active, inactive, placed)".into(), default: Some(serde_json::json!("active")) },
+                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
+                ],
+                returns: "List of candidates with id, name, phone, email, skills, experience".into(),
+                sql_template: "SELECT candidate_id, first_name, last_name, phone, email, city, state, zip, vertical, skills, years_experience FROM candidates WHERE 1=1 {skills_filter} {city_filter} {state_filter} {years_filter} {status_filter} ORDER BY years_experience DESC LIMIT {limit}".into(),
+                category: "candidates".into(),
+            },
+            ToolDef {
+                name: "get_candidate".into(),
+                description: "Get full details for a specific candidate by ID.".into(),
+                permission: Permission::Read,
+                parameters: vec![
+                    ParamDef { name: "candidate_id".into(), param_type: "string".into(), required: true, description: "Candidate ID (e.g. CAND-000001)".into(), default: None },
+                ],
+                returns: "Full candidate record".into(),
+                sql_template: "SELECT * FROM candidates WHERE candidate_id = '{candidate_id}'".into(),
+                category: "candidates".into(),
+            },
+            ToolDef {
+                name: "revenue_by_client".into(),
+                description: "Show total billed revenue, pay costs, and gross profit by client. Filter by date range.".into(),
+                permission: Permission::Read,
+                parameters: vec![
+                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Top N clients".into(), default: Some(serde_json::json!(10)) },
+                ],
+                returns: "Client name, total billed, total paid, gross profit, timesheet count".into(),
+                sql_template: "SELECT c.company_name, COUNT(*) as timesheets, ROUND(SUM(t.bill_total),2) as total_billed, ROUND(SUM(t.pay_total),2) as total_paid, ROUND(SUM(t.bill_total) - SUM(t.pay_total),2) as gross_profit FROM timesheets t JOIN clients c ON t.client_id = c.client_id WHERE t.approved = true GROUP BY c.company_name ORDER BY total_billed DESC LIMIT {limit}".into(),
+                category: "analytics".into(),
+            },
+            ToolDef {
+                name: "recruiter_performance".into(),
+                description: "Show recruiter performance: placements, unique candidates, and total revenue generated.".into(),
+                permission: Permission::Read,
+                parameters: vec![
+                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Top N recruiters".into(), default: Some(serde_json::json!(10)) },
+                ],
+                returns: "Recruiter name, placement count, unique candidates, total revenue".into(),
+                sql_template: "SELECT p.recruiter, COUNT(DISTINCT p.placement_id) as placements, COUNT(DISTINCT p.candidate_id) as unique_candidates, ROUND(SUM(t.bill_total),2) as total_revenue FROM placements p JOIN timesheets t ON p.placement_id = t.placement_id GROUP BY p.recruiter ORDER BY total_revenue DESC LIMIT {limit}".into(),
+                category: "analytics".into(),
+            },
+            ToolDef {
+                name: "cold_leads".into(),
+                description: "Find candidates who were called multiple times but never placed — potential lost opportunities.".into(),
+                permission: Permission::Read,
+                parameters: vec![
+                    ParamDef { name: "min_calls".into(), param_type: "integer".into(), required: false, description: "Minimum call count".into(), default: Some(serde_json::json!(5)) },
+                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
+                ],
+                returns: "Candidates with call counts who were never placed".into(),
+                sql_template: "SELECT c.candidate_id, c.first_name, c.last_name, c.phone, c.vertical, cl.calls FROM candidates c JOIN (SELECT candidate_id, COUNT(*) as calls FROM call_log GROUP BY candidate_id HAVING COUNT(*) >= {min_calls}) cl ON c.candidate_id = cl.candidate_id WHERE c.candidate_id NOT IN (SELECT DISTINCT candidate_id FROM placements) ORDER BY cl.calls DESC LIMIT {limit}".into(),
+                category: "analytics".into(),
+            },
+            ToolDef {
+                name: "open_jobs".into(),
+                description: "List open job orders with client, title, rates, and location.".into(),
+                permission: Permission::Read,
+                parameters: vec![
+                    ParamDef { name: "vertical".into(), param_type: "string".into(), required: false, description: "Filter by vertical (IT, Healthcare, etc)".into(), default: None },
+                    ParamDef { name: "city".into(), param_type: "string".into(), required: false, description: "Filter by city".into(), default: None },
+                    ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
+                ],
+                returns: "Open job orders with details".into(),
+                sql_template: "SELECT j.job_order_id, c.company_name, j.title, j.vertical, j.city, j.state, j.bill_rate, j.pay_rate, j.status FROM job_orders j JOIN clients c ON j.client_id = c.client_id WHERE j.status = 'open' {vertical_filter} {city_filter} ORDER BY j.bill_rate DESC LIMIT {limit}".into(),
+                category: "jobs".into(),
+            },
+        ];
+
+        let mut reg = self.tools.write().await;
+        for tool in tools {
+            reg.insert(tool.name.clone(), tool);
+        }
+    }
+
+    /// Get tool definition (for LLM tool-use schema).
+    pub async fn get_tool(&self, name: &str) -> Option<ToolDef> {
+        self.tools.read().await.get(name).cloned()
+    }
+
+    /// List all tools (for LLM tool discovery).
+    pub async fn list_tools(&self) -> Vec<ToolDef> {
+        self.tools.read().await.values().cloned().collect()
+    }
+
+    /// Build SQL from tool parameters. Validates and sanitizes.
+    pub fn build_sql(tool: &ToolDef, params: &serde_json::Value) -> Result<String, String> {
+        let mut sql = tool.sql_template.clone();
+        let params = params.as_object().ok_or("params must be an object")?;
+
+        // Replace named parameters
+        for param_def in &tool.parameters {
+            let value = params.get(&param_def.name)
+                .or(param_def.default.as_ref());
+
+            match param_def.name.as_str() {
+                // Handle filter parameters (conditional WHERE clauses)
+                "skills" => {
+                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+                        let filters: Vec<String> = v.split(',')
+                            .map(|s| format!("skills LIKE '%{}%'", s.trim().replace('\'', "''")))
+                            .collect();
+                        sql = sql.replace("{skills_filter}", &format!("AND ({})", filters.join(" OR ")));
+                    } else {
+                        sql = sql.replace("{skills_filter}", "");
+                    }
+                }
+                "city" => {
+                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+                        sql = sql.replace("{city_filter}", &format!("AND city = '{}'", v.replace('\'', "''")));
+                    } else {
+                        sql = sql.replace("{city_filter}", "");
+                    }
+                }
+                "state" => {
+                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+                        sql = sql.replace("{state_filter}", &format!("AND state = '{}'", v.replace('\'', "''")));
+                    } else {
+                        sql = sql.replace("{state_filter}", "");
+                    }
+                }
+                "vertical" => {
+                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+                        sql = sql.replace("{vertical_filter}", &format!("AND j.vertical = '{}'", v.replace('\'', "''")));
+                    } else {
+                        sql = sql.replace("{vertical_filter}", "");
+                    }
+                }
+                "status" => {
+                    if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+                        sql = sql.replace("{status_filter}", &format!("AND status = '{}'", v.replace('\'', "''")));
+                    } else {
+                        sql = sql.replace("{status_filter}", "");
+                    }
+                }
+                "min_years" => {
+                    if let Some(v) = value.and_then(|v| v.as_i64()) {
+                        sql = sql.replace("{years_filter}", &format!("AND years_experience >= {v}"));
+                    } else {
+                        sql = sql.replace("{years_filter}", "");
+                    }
+                }
+                // Direct substitution for simple params
+                _ => {
+                    let placeholder = format!("{{{}}}", param_def.name);
+                    if sql.contains(&placeholder) {
+                        let val_str = match value {
+                            Some(serde_json::Value::String(s)) => s.replace('\'', "''"),
+                            Some(serde_json::Value::Number(n)) => n.to_string(),
+                            Some(serde_json::Value::Bool(b)) => b.to_string(),
+                            Some(v) => v.to_string(),
+                            None if param_def.required => return Err(format!("missing required param: {}", param_def.name)),
+                            None => continue,
+                        };
+                        sql = sql.replace(&placeholder, &val_str);
+                    }
+                }
+            }
+        }
+
+        Ok(sql)
+    }
+
+    /// Log a tool invocation.
+    pub async fn log_invocation(&self, inv: ToolInvocation) {
+        self.audit_log.write().await.push(inv);
+    }
+
+    /// Get recent audit log.
+    pub async fn recent_audit(&self, limit: usize) -> Vec<ToolInvocation> {
+        let log = self.audit_log.read().await;
+        let start = log.len().saturating_sub(limit);
+        log[start..].iter().rev().cloned().collect()
+    }
+}
@@ -0,0 +1,148 @@
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use serde::Deserialize;
+
+use super::registry::{Permission, ToolInvocation, ToolRegistry};
+use crate::tools::ToolState;
+
+pub fn router(state: ToolState) -> Router {
+    Router::new()
+        .route("/", get(list_tools))
+        .route("/{name}", get(get_tool))
+        .route("/{name}/call", post(call_tool))
+        .route("/audit", get(audit_log))
+        .with_state(state)
+}
+
+/// List all available tools (for LLM tool-use discovery).
+async fn list_tools(State(state): State<ToolState>) -> impl IntoResponse {
+    let tools = state.registry.list_tools().await;
+    // Return in MCP-compatible format
+    let tool_list: Vec<serde_json::Value> = tools.iter().map(|t| {
+        serde_json::json!({
+            "name": t.name,
+            "description": t.description,
+            "permission": t.permission,
+            "category": t.category,
+            "parameters": t.parameters.iter().map(|p| {
+                serde_json::json!({
+                    "name": p.name,
+                    "type": p.param_type,
+                    "required": p.required,
+                    "description": p.description,
+                    "default": p.default,
+                })
+            }).collect::<Vec<_>>(),
+        })
+    }).collect();
+    Json(tool_list)
+}
+
+/// Get a specific tool definition.
+async fn get_tool(
+    State(state): State<ToolState>,
+    Path(name): Path<String>,
+) -> impl IntoResponse {
+    match state.registry.get_tool(&name).await {
+        Some(tool) => Ok(Json(tool)),
+        None => Err((StatusCode::NOT_FOUND, format!("tool not found: {name}"))),
+    }
+}
+
+/// Call a tool — validate params, execute SQL, log invocation.
+#[derive(Deserialize)]
+struct CallRequest {
+    params: serde_json::Value,
+    agent: String,
+}
+
+async fn call_tool(
+    State(state): State<ToolState>,
+    Path(name): Path<String>,
+    Json(req): Json<CallRequest>,
+) -> impl IntoResponse {
+    let tool = match state.registry.get_tool(&name).await {
+        Some(t) => t,
+        None => return Err((StatusCode::NOT_FOUND, format!("tool not found: {name}"))),
+    };
+
+    tracing::info!("tool call: {} by agent '{}' with {:?}", name, req.agent, req.params);
+
+    // Build SQL from params
+    let sql = match ToolRegistry::build_sql(&tool, &req.params) {
+        Ok(s) => s,
+        Err(e) => {
+            state.registry.log_invocation(ToolInvocation {
+                id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
+                tool_name: name.clone(),
+                agent: req.agent.clone(),
+                params: req.params.clone(),
+                permission: tool.permission.clone(),
+                timestamp: chrono::Utc::now(),
+                success: false,
+                error: Some(e.clone()),
+                rows_returned: None,
+            }).await;
+            return Err((StatusCode::BAD_REQUEST, e));
+        }
+    };
+
+    // Execute via query engine
+    let result = state.query_fn.execute(&sql).await;
+
+    match result {
+        Ok((rows, row_count)) => {
+            state.registry.log_invocation(ToolInvocation {
+                id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
+                tool_name: name.clone(),
+                agent: req.agent,
+                params: req.params,
+                permission: tool.permission,
+                timestamp: chrono::Utc::now(),
+                success: true,
+                error: None,
+                rows_returned: Some(row_count),
+            }).await;
+
+            Ok(Json(serde_json::json!({
+                "tool": name,
+                "rows": rows,
+                "row_count": row_count,
+                "sql": sql,
+            })))
+        }
+        Err(e) => {
+            state.registry.log_invocation(ToolInvocation {
+                id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
+                tool_name: name,
+                agent: req.agent,
+                params: req.params,
+                permission: tool.permission,
+                timestamp: chrono::Utc::now(),
+                success: false,
+                error: Some(e.clone()),
+                rows_returned: None,
+            }).await;
+
+            Err((StatusCode::INTERNAL_SERVER_ERROR, e))
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct AuditQuery {
+    limit: Option<usize>,
+}
+
+async fn audit_log(
+    State(state): State<ToolState>,
+    Query(q): Query<AuditQuery>,
+) -> impl IntoResponse {
+    let log = state.registry.recent_audit(q.limit.unwrap_or(50)).await;
+    Json(log)
+}
@@ -0,0 +1,23 @@
+[package]
+name = "ingestd"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+shared = { path = "../shared" }
+storaged = { path = "../storaged" }
+catalogd = { path = "../catalogd" }
+tokio = { workspace = true }
+axum = { workspace = true, features = ["multipart"] }
+lopdf = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tracing = { workspace = true }
+arrow = { workspace = true }
+parquet = { workspace = true }
+bytes = { workspace = true }
+sha2 = { workspace = true }
+csv = { workspace = true }
+chrono = { workspace = true }
+object_store = { workspace = true }
+tokio-postgres = { workspace = true }
@@ -0,0 +1,235 @@
+/// ChatGPT PDF conversation parser.
+/// Extracts structured conversation turns from ChatGPT export PDFs.
+/// Handles: user/assistant turns, timestamps, conversation titles.
+
+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use std::sync::Arc;
+
+/// A single conversation turn.
+#[derive(Debug, Clone)]
+pub struct ConversationTurn {
+    pub source_file: String,
+    pub conversation_title: String,
+    pub turn_number: i32,
+    pub role: String,        // "user" or "assistant"
+    pub content: String,
+    pub page_number: i32,
+}
+
+/// Parse a ChatGPT PDF into structured conversation turns.
+pub fn parse_chatgpt_pdf(content: &[u8], filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
+    let doc = lopdf::Document::load_mem(content)
+        .map_err(|e| format!("PDF load error: {e}"))?;
+
+    // Extract all text
+    let pages = doc.get_pages();
+    let mut all_text = String::new();
+    let mut page_texts: Vec<(i32, String)> = Vec::new();
+
+    for (&page_num, _) in pages.iter() {
+        let text = doc.extract_text(&[page_num]).unwrap_or_default();
+        let text = text.trim().to_string();
+        if !text.is_empty() {
+            page_texts.push((page_num as i32, text.clone()));
+            all_text.push_str(&text);
+            all_text.push('\n');
+        }
+    }
+
+    if all_text.trim().is_empty() {
+        return Err("PDF contains no extractable text".into());
+    }
+
+    // Try to parse as ChatGPT conversation format
+    let turns = extract_turns(&all_text, filename, &page_texts);
+
+    if turns.is_empty() {
+        // Fall back to page-based extraction if we can't detect conversation structure
+        return parse_as_raw_pages(&page_texts, filename);
+    }
+
+    tracing::info!("parsed ChatGPT PDF '{}': {} turns", filename, turns.len());
+    turns_to_batch(&turns)
+}
+
+/// Extract conversation turns from ChatGPT PDF text.
+/// ChatGPT PDFs typically have patterns like:
+/// - "You said:" / "ChatGPT said:" (newer format)
+/// - "User:" / "Assistant:"
+/// - Or just alternating blocks separated by formatting
+fn extract_turns(text: &str, filename: &str, page_texts: &[(i32, String)]) -> Vec<ConversationTurn> {
+    let mut turns = Vec::new();
+    let mut turn_num = 0;
+
+    // Try to extract conversation title from first line
+    let title = text.lines().next()
+        .map(|l| l.trim().to_string())
+        .filter(|l| l.len() < 200 && !l.is_empty())
+        .unwrap_or_else(|| filename.replace(".pdf", ""));
+
+    // Strategy 1: Look for "You said:" / "ChatGPT said:" markers (common format)
+    let you_markers = ["You said:", "You:", "User:", "Human:"];
+    let ai_markers = ["ChatGPT said:", "ChatGPT:", "Assistant:", "AI:"];
+
+    let has_markers = you_markers.iter().any(|m| text.contains(m))
+        && ai_markers.iter().any(|m| text.contains(m));
+
+    if has_markers {
+        let lines: Vec<&str> = text.lines().collect();
+        let mut current_role = String::new();
+        let mut current_content = String::new();
+        let mut current_page = 1i32;
+
+        for line in &lines {
+            let trimmed = line.trim();
+
+            // Check for role markers
+            let mut found_marker = false;
+            for marker in &you_markers {
+                if trimmed.starts_with(marker) || trimmed == *marker {
+                    // Save previous turn
+                    if !current_content.trim().is_empty() && !current_role.is_empty() {
+                        turn_num += 1;
+                        turns.push(ConversationTurn {
+                            source_file: filename.to_string(),
+                            conversation_title: title.clone(),
+                            turn_number: turn_num,
+                            role: current_role.clone(),
+                            content: current_content.trim().to_string(),
+                            page_number: current_page,
+                        });
+                    }
+                    current_role = "user".to_string();
+                    current_content = trimmed.strip_prefix(marker).unwrap_or("").trim().to_string();
+                    current_content.push('\n');
+                    found_marker = true;
+                    break;
+                }
+            }
+            if !found_marker {
+                for marker in &ai_markers {
+                    if trimmed.starts_with(marker) || trimmed == *marker {
+                        if !current_content.trim().is_empty() && !current_role.is_empty() {
+                            turn_num += 1;
+                            turns.push(ConversationTurn {
+                                source_file: filename.to_string(),
+                                conversation_title: title.clone(),
+                                turn_number: turn_num,
+                                role: current_role.clone(),
+                                content: current_content.trim().to_string(),
+                                page_number: current_page,
+                            });
+                        }
+                        current_role = "assistant".to_string();
+                        current_content = trimmed.strip_prefix(marker).unwrap_or("").trim().to_string();
+                        current_content.push('\n');
+                        found_marker = true;
+                        break;
+                    }
+                }
+            }
+
+            if !found_marker && !current_role.is_empty() {
+                current_content.push_str(trimmed);
+                current_content.push('\n');
+            }
+        }
+
+        // Save last turn
+        if !current_content.trim().is_empty() && !current_role.is_empty() {
+            turn_num += 1;
+            turns.push(ConversationTurn {
+                source_file: filename.to_string(),
+                conversation_title: title.clone(),
+                turn_number: turn_num,
+                role: current_role,
+                content: current_content.trim().to_string(),
+                page_number: current_page,
+            });
+        }
+
+        return turns;
+    }
+
+    // Strategy 2: Split by paragraph blocks, alternate user/assistant
+    // This is a heuristic fallback for PDFs without clear markers
+    let paragraphs: Vec<&str> = text.split("\n\n")
+        .map(|p| p.trim())
+        .filter(|p| p.len() > 20) // skip short fragments
+        .collect();
+
+    if paragraphs.len() >= 2 {
+        for (i, para) in paragraphs.iter().enumerate() {
+            turn_num += 1;
+            turns.push(ConversationTurn {
+                source_file: filename.to_string(),
+                conversation_title: title.clone(),
+                turn_number: turn_num,
+                role: if i % 2 == 0 { "user" } else { "assistant" }.to_string(),
+                content: para.to_string(),
+                page_number: 1,
+            });
+        }
+    }
+
+    turns
+}
+
+/// Fallback: parse as raw pages with full text.
+fn parse_as_raw_pages(page_texts: &[(i32, String)], filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("source_file", DataType::Utf8, false),
+        Field::new("conversation_title", DataType::Utf8, false),
+        Field::new("turn_number", DataType::Int32, false),
+        Field::new("role", DataType::Utf8, false),
+        Field::new("content", DataType::Utf8, false),
+        Field::new("page_number", DataType::Int32, false),
+    ]));
+
+    let sources: Vec<&str> = page_texts.iter().map(|_| filename).collect();
+    let titles: Vec<String> = page_texts.iter().map(|_| filename.replace(".pdf", "")).collect();
+    let turn_nums: Vec<i32> = (1..=page_texts.len() as i32).collect();
+    let roles: Vec<&str> = page_texts.iter().map(|_| "page").collect();
+    let contents: Vec<&str> = page_texts.iter().map(|(_, t)| t.as_str()).collect();
+    let pages: Vec<i32> = page_texts.iter().map(|(p, _)| *p).collect();
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(StringArray::from(sources)),
+        Arc::new(StringArray::from(titles.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
+        Arc::new(Int32Array::from(turn_nums)),
+        Arc::new(StringArray::from(roles)),
+        Arc::new(StringArray::from(contents)),
+        Arc::new(Int32Array::from(pages)),
+    ];
+
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    Ok((schema, vec![batch]))
+}
+
+fn turns_to_batch(turns: &[ConversationTurn]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("source_file", DataType::Utf8, false),
+        Field::new("conversation_title", DataType::Utf8, false),
+        Field::new("turn_number", DataType::Int32, false),
+        Field::new("role", DataType::Utf8, false),
+        Field::new("content", DataType::Utf8, false),
+        Field::new("page_number", DataType::Int32, false),
+    ]));
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(StringArray::from(turns.iter().map(|t| t.source_file.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(turns.iter().map(|t| t.conversation_title.as_str()).collect::<Vec<_>>())),
+        Arc::new(Int32Array::from(turns.iter().map(|t| t.turn_number).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(turns.iter().map(|t| t.role.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(turns.iter().map(|t| t.content.as_str()).collect::<Vec<_>>())),
+        Arc::new(Int32Array::from(turns.iter().map(|t| t.page_number).collect::<Vec<_>>())),
+    ];
+
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    Ok((schema, vec![batch]))
+}
@@ -0,0 +1,203 @@
+use arrow::array::{ArrayRef, Float64Array, Int64Array, StringArray, BooleanArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::record_batch::RecordBatch;
+use std::sync::Arc;
+
+/// Inferred column type from sampling data.
+#[derive(Debug, Clone, PartialEq)]
+enum InferredType {
+    Integer,
+    Float,
+    Boolean,
+    String,
+}
+
+/// Parse CSV bytes into Arrow RecordBatches with automatic schema detection.
+/// Per ADR-010: ambiguous types default to String.
+pub fn parse_csv(content: &[u8]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
+    let mut reader = csv::ReaderBuilder::new()
+        .flexible(true) // allow varying column counts
+        .trim(csv::Trim::All)
+        .from_reader(content);
+
+    let headers: Vec<String> = reader.headers()
+        .map_err(|e| format!("CSV header error: {e}"))?
+        .iter()
+        .enumerate()
+        .map(|(i, h)| {
+            let h = h.trim().to_string();
+            if h.is_empty() { format!("column_{i}") } else { sanitize_column_name(&h) }
+        })
+        .collect();
+
+    let n_cols = headers.len();
+    if n_cols == 0 {
+        return Err("CSV has no columns".into());
+    }
+
+    // Read all rows into string columns
+    let mut columns: Vec<Vec<String>> = vec![vec![]; n_cols];
+    let mut row_count = 0;
+
+    for result in reader.records() {
+        let record = result.map_err(|e| format!("CSV row error: {e}"))?;
+        for (i, field) in record.iter().enumerate() {
+            if i < n_cols {
+                columns[i].push(field.trim().to_string());
+            }
+        }
+        // Pad short rows with empty strings
+        for col in columns.iter_mut().skip(record.len().min(n_cols)) {
+            col.push(String::new());
+        }
+        row_count += 1;
+    }
+
+    if row_count == 0 {
+        return Err("CSV has no data rows".into());
+    }
+
+    tracing::info!("parsed CSV: {row_count} rows × {n_cols} columns");
+
+    // Infer types by sampling (look at all values)
+    let types: Vec<InferredType> = columns.iter().map(|col| infer_column_type(col)).collect();
+
+    // Build Arrow schema
+    let fields: Vec<Field> = headers.iter().zip(types.iter()).map(|(name, typ)| {
+        let dt = match typ {
+            InferredType::Integer => DataType::Int64,
+            InferredType::Float => DataType::Float64,
+            InferredType::Boolean => DataType::Boolean,
+            InferredType::String => DataType::Utf8,
+        };
+        Field::new(name, dt, true) // all nullable
+    }).collect();
+
+    let schema = Arc::new(Schema::new(fields));
+
+    // Build arrays
+    let arrays: Vec<ArrayRef> = columns.iter().zip(types.iter()).map(|(col, typ)| {
+        match typ {
+            InferredType::Integer => {
+                let vals: Vec<Option<i64>> = col.iter().map(|v| {
+                    if v.is_empty() { None } else { v.replace(',', "").parse().ok() }
+                }).collect();
+                Arc::new(Int64Array::from(vals)) as ArrayRef
+            }
+            InferredType::Float => {
+                let vals: Vec<Option<f64>> = col.iter().map(|v| {
+                    if v.is_empty() { None }
+                    else { v.replace(',', "").replace('$', "").replace('%', "").parse().ok() }
+                }).collect();
+                Arc::new(Float64Array::from(vals)) as ArrayRef
+            }
+            InferredType::Boolean => {
+                let vals: Vec<Option<bool>> = col.iter().map(|v| {
+                    match v.to_lowercase().as_str() {
+                        "true" | "yes" | "1" | "y" | "t" => Some(true),
+                        "false" | "no" | "0" | "n" | "f" => Some(false),
+                        _ => None,
+                    }
+                }).collect();
+                Arc::new(BooleanArray::from(vals)) as ArrayRef
+            }
+            InferredType::String => {
+                Arc::new(StringArray::from(col.clone())) as ArrayRef
+            }
+        }
+    }).collect();
+
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    Ok((schema, vec![batch]))
+}
+
+/// Infer column type from values. Conservative: defaults to String on ambiguity.
+fn infer_column_type(values: &[String]) -> InferredType {
+    let non_empty: Vec<&str> = values.iter()
+        .map(|v| v.as_str())
+        .filter(|v| !v.is_empty() && *v != "NULL" && *v != "null" && *v != "N/A" && *v != "n/a")
+        .collect();
+
+    if non_empty.is_empty() {
+        return InferredType::String;
+    }
+
+    // Check boolean
+    let all_bool = non_empty.iter().all(|v| {
+        matches!(v.to_lowercase().as_str(), "true" | "false" | "yes" | "no" | "1" | "0" | "y" | "n" | "t" | "f")
+    });
+    if all_bool && non_empty.len() >= 2 {
+        // Make sure it's not just all "1" and "0" which could be integers
+        let has_text_bool = non_empty.iter().any(|v| {
+            matches!(v.to_lowercase().as_str(), "true" | "false" | "yes" | "no" | "y" | "n" | "t" | "f")
+        });
+        if has_text_bool {
+            return InferredType::Boolean;
+        }
+    }
+
+    // Check integer (allow commas as thousands separator)
+    let int_rate = non_empty.iter()
+        .filter(|v| v.replace(',', "").parse::<i64>().is_ok())
+        .count() as f64 / non_empty.len() as f64;
+    if int_rate > 0.95 {
+        return InferredType::Integer;
+    }
+
+    // Check float (allow $, %, commas)
+    let float_rate = non_empty.iter()
+        .filter(|v| v.replace(',', "").replace('$', "").replace('%', "").parse::<f64>().is_ok())
+        .count() as f64 / non_empty.len() as f64;
+    if float_rate > 0.95 {
+        return InferredType::Float;
+    }
+
+    InferredType::String
+}
+
+/// Sanitize column name for SQL compatibility.
+fn sanitize_column_name(name: &str) -> String {
+    name.chars()
+        .map(|c| if c.is_alphanumeric() || c == '_' { c.to_ascii_lowercase() } else { '_' })
+        .collect::<String>()
+        .trim_matches('_')
+        .to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_simple_csv() {
+        let csv = b"Name,Age,Salary\nAlice,30,50000\nBob,25,45000\n";
+        let (schema, batches) = parse_csv(csv).unwrap();
+        assert_eq!(schema.fields().len(), 3);
+        assert_eq!(batches[0].num_rows(), 2);
+        assert_eq!(schema.field(1).data_type(), &DataType::Int64);
+        assert_eq!(schema.field(2).data_type(), &DataType::Int64);
+    }
+
+    #[test]
+    fn parse_csv_with_mixed_types() {
+        let csv = b"id,value\n1,hello\n2,world\n3,N/A\n";
+        let (schema, _) = parse_csv(csv).unwrap();
+        assert_eq!(schema.field(0).data_type(), &DataType::Int64);
+        assert_eq!(schema.field(1).data_type(), &DataType::Utf8);
+    }
+
+    #[test]
+    fn parse_csv_with_dollar_amounts() {
+        let csv = b"item,price\nWidget,$29.99\nGadget,$149.50\n";
+        let (schema, _) = parse_csv(csv).unwrap();
+        assert_eq!(schema.field(1).data_type(), &DataType::Float64);
+    }
+
+    #[test]
+    fn sanitize_names() {
+        assert_eq!(sanitize_column_name("First Name"), "first_name");
+        assert_eq!(sanitize_column_name("Bill Rate ($)"), "bill_rate");
+    }
+}
@@ -0,0 +1,225 @@
+/// Database connector: pull tables from PostgreSQL directly into Parquet.
+/// Connects, reads schema, streams rows into Arrow RecordBatches, writes Parquet.
+/// No ORM, no schema definition — schema inferred from the database.
+
+use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, StringArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::record_batch::RecordBatch;
+use std::sync::Arc;
+use tokio_postgres::{Client, NoTls, types::Type as PgType};
+
+/// Connection config for a database.
+#[derive(Debug, Clone, serde::Deserialize)]
+pub struct DbConfig {
+    pub host: String,
+    pub port: u16,
+    pub database: String,
+    pub user: String,
+    pub password: String,
+}
+
+impl DbConfig {
+    pub fn connection_string(&self) -> String {
+        if self.password.is_empty() {
+            format!(
+                "host={} port={} dbname={} user={}",
+                self.host, self.port, self.database, self.user
+            )
+        } else {
+            format!(
+                "host={} port={} dbname={} user={} password={}",
+                self.host, self.port, self.database, self.user, self.password
+            )
+        }
+    }
+}
+
+/// Result of a database table import.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct DbIngestResult {
+    pub table_name: String,
+    pub rows: usize,
+    pub columns: usize,
+    pub schema_detected: Vec<String>, // column name: type pairs
+}
+
+/// Connect to PostgreSQL and import a table as Arrow RecordBatches.
+pub async fn import_postgres_table(
+    config: &DbConfig,
+    table_name: &str,
+    limit: Option<usize>,
+) -> Result<(Arc<Schema>, Vec<RecordBatch>, DbIngestResult), String> {
+    // Connect
+    let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
+        .await
+        .map_err(|e| format!("postgres connect error: {e}"))?;
+
+    // Spawn connection handler
+    tokio::spawn(async move {
+        if let Err(e) = connection.await {
+            tracing::error!("postgres connection error: {e}");
+        }
+    });
+
+    tracing::info!("connected to postgres {}:{}/{}", config.host, config.port, config.database);
+
+    // Get column info
+    let columns = get_table_columns(&client, table_name).await?;
+    if columns.is_empty() {
+        return Err(format!("table '{}' not found or has no columns", table_name));
+    }
+
+    let schema_detected: Vec<String> = columns.iter()
+        .map(|(name, pg_type)| format!("{}: {}", name, pg_type_to_string(pg_type)))
+        .collect();
+
+    tracing::info!("table '{}': {} columns", table_name, columns.len());
+
+    // Build Arrow schema
+    let arrow_fields: Vec<Field> = columns.iter()
+        .map(|(name, pg_type)| Field::new(name, pg_type_to_arrow(pg_type), true))
+        .collect();
+    let schema = Arc::new(Schema::new(arrow_fields));
+
+    // Query rows
+    let sql = match limit {
+        Some(n) => format!("SELECT * FROM \"{}\" LIMIT {}", table_name, n),
+        None => format!("SELECT * FROM \"{}\"", table_name),
+    };
+
+    let rows = client.query(&sql, &[])
+        .await
+        .map_err(|e| format!("query error: {e}"))?;
+
+    let row_count = rows.len();
+    tracing::info!("fetched {} rows from '{}'", row_count, table_name);
+
+    if row_count == 0 {
+        return Ok((schema.clone(), vec![], DbIngestResult {
+            table_name: table_name.to_string(),
+            rows: 0,
+            columns: columns.len(),
+            schema_detected,
+        }));
+    }
+
+    // Convert to Arrow arrays
+    let mut arrays: Vec<ArrayRef> = Vec::new();
+
+    for (col_idx, (_, pg_type)) in columns.iter().enumerate() {
+        let array = rows_to_arrow_column(&rows, col_idx, pg_type)?;
+        arrays.push(array);
+    }
+
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    let result = DbIngestResult {
+        table_name: table_name.to_string(),
+        rows: row_count,
+        columns: columns.len(),
+        schema_detected,
+    };
+
+    Ok((schema, vec![batch], result))
+}
+
+/// List all tables in the database.
+pub async fn list_postgres_tables(config: &DbConfig) -> Result<Vec<String>, String> {
+    let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
+        .await
+        .map_err(|e| format!("connect error: {e}"))?;
+
+    tokio::spawn(async move { let _ = connection.await; });
+
+    let rows = client.query(
+        "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name",
+        &[],
+    ).await.map_err(|e| format!("query error: {e}"))?;
+
+    Ok(rows.iter().map(|r| r.get::<_, String>(0)).collect())
+}
+
+/// Get column names and types for a table.
+async fn get_table_columns(client: &Client, table_name: &str) -> Result<Vec<(String, PgType)>, String> {
+    // Use a dummy query to get column types
+    let stmt = client.prepare(&format!("SELECT * FROM \"{}\" LIMIT 0", table_name))
+        .await
+        .map_err(|e| format!("prepare error for '{}': {e}", table_name))?;
+
+    Ok(stmt.columns().iter()
+        .map(|col| (col.name().to_string(), col.type_().clone()))
+        .collect())
+}
+
+/// Convert a Postgres type to Arrow DataType.
+fn pg_type_to_arrow(pg_type: &PgType) -> DataType {
+    match *pg_type {
+        PgType::BOOL => DataType::Boolean,
+        PgType::INT2 | PgType::INT4 => DataType::Int32,
+        PgType::INT8 | PgType::OID => DataType::Int64,
+        PgType::FLOAT4 | PgType::FLOAT8 | PgType::NUMERIC => DataType::Float64,
+        _ => DataType::Utf8, // everything else → string (safe default per ADR-010)
+    }
+}
+
+fn pg_type_to_string(pg_type: &PgType) -> String {
+    format!("{}", pg_type)
+}
+
+/// Convert a column of Postgres rows to an Arrow array.
+fn rows_to_arrow_column(
+    rows: &[tokio_postgres::Row],
+    col_idx: usize,
+    pg_type: &PgType,
+) -> Result<ArrayRef, String> {
+    match *pg_type {
+        PgType::BOOL => {
+            let vals: Vec<Option<bool>> = rows.iter()
+                .map(|r| r.try_get(col_idx).ok())
+                .collect();
+            Ok(Arc::new(BooleanArray::from(vals)))
+        }
+        PgType::INT2 => {
+            let vals: Vec<Option<i32>> = rows.iter()
+                .map(|r| r.try_get::<_, i16>(col_idx).ok().map(|v| v as i32))
+                .collect();
+            Ok(Arc::new(Int32Array::from(vals)))
+        }
+        PgType::INT4 => {
+            let vals: Vec<Option<i32>> = rows.iter()
+                .map(|r| r.try_get(col_idx).ok())
+                .collect();
+            Ok(Arc::new(Int32Array::from(vals)))
+        }
+        PgType::INT8 | PgType::OID => {
+            let vals: Vec<Option<i64>> = rows.iter()
+                .map(|r| r.try_get(col_idx).ok())
+                .collect();
+            Ok(Arc::new(Int64Array::from(vals)))
+        }
+        PgType::FLOAT4 => {
+            let vals: Vec<Option<f64>> = rows.iter()
+                .map(|r| r.try_get::<_, f32>(col_idx).ok().map(|v| v as f64))
+                .collect();
+            Ok(Arc::new(Float64Array::from(vals)))
+        }
+        PgType::FLOAT8 => {
+            let vals: Vec<Option<f64>> = rows.iter()
+                .map(|r| r.try_get(col_idx).ok())
+                .collect();
+            Ok(Arc::new(Float64Array::from(vals)))
+        }
+        _ => {
+            // Default: try to get as string
+            let vals: Vec<Option<String>> = rows.iter()
+                .map(|r| {
+                    r.try_get::<_, String>(col_idx).ok()
+                        .or_else(|| r.try_get::<_, serde_json::Value>(col_idx).ok().map(|v| v.to_string()))
+                        .or_else(|| Some("".to_string()))
+                })
+                .collect();
+            Ok(Arc::new(StringArray::from(vals)))
+        }
+    }
+}
@@ -0,0 +1,103 @@
+use sha2::{Digest, Sha256};
+
+/// Detected file type from content inspection.
+#[derive(Debug, Clone, PartialEq)]
+pub enum FileType {
+    Csv,
+    Json,
+    NdJson, // newline-delimited JSON
+    Pdf,
+    Text,   // plain text, SMS logs, etc.
+    Unknown,
+}
+
+/// Detect file type from filename extension and content sniffing.
+pub fn detect_file_type(filename: &str, content: &[u8]) -> FileType {
+    // Extension-based first
+    let lower = filename.to_lowercase();
+    if lower.ends_with(".csv") || lower.ends_with(".tsv") {
+        return FileType::Csv;
+    }
+    if lower.ends_with(".json") {
+        // Check if it's newline-delimited JSON
+        if content.iter().take(4096).filter(|&&b| b == b'\n').count() > 2 {
+            let first_line = content.split(|&b| b == b'\n').next().unwrap_or(b"");
+            if first_line.starts_with(b"{") {
+                return FileType::NdJson;
+            }
+        }
+        return FileType::Json;
+    }
+    if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") {
+        return FileType::NdJson;
+    }
+    if lower.ends_with(".pdf") {
+        return FileType::Pdf;
+    }
+    if lower.ends_with(".txt") || lower.ends_with(".log") || lower.ends_with(".sms") {
+        return FileType::Text;
+    }
+
+    // Content sniffing fallback
+    if content.starts_with(b"%PDF") {
+        return FileType::Pdf;
+    }
+    if content.starts_with(b"[") || content.starts_with(b"{") {
+        return FileType::Json;
+    }
+
+    // Check if it looks like CSV (has commas and newlines in first chunk)
+    let sample = &content[..content.len().min(4096)];
+    let comma_count = sample.iter().filter(|&&b| b == b',').count();
+    let newline_count = sample.iter().filter(|&&b| b == b'\n').count();
+    if comma_count > 3 && newline_count > 1 {
+        return FileType::Csv;
+    }
+
+    // If it's valid UTF-8, treat as text
+    if std::str::from_utf8(sample).is_ok() {
+        return FileType::Text;
+    }
+
+    FileType::Unknown
+}
+
+/// Compute SHA-256 hash of content for deduplication.
+pub fn content_hash(content: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(content);
+    format!("{:x}", hasher.finalize())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn detect_csv_by_extension() {
+        assert_eq!(detect_file_type("data.csv", b"a,b,c\n1,2,3"), FileType::Csv);
+    }
+
+    #[test]
+    fn detect_json_by_extension() {
+        assert_eq!(detect_file_type("data.json", b"[{\"a\":1}]"), FileType::Json);
+    }
+
+    #[test]
+    fn detect_pdf_by_magic() {
+        assert_eq!(detect_file_type("unknown", b"%PDF-1.4 blah"), FileType::Pdf);
+    }
+
+    #[test]
+    fn detect_csv_by_content() {
+        let csv = b"name,age,city\nAlice,30,NYC\nBob,25,LA\n";
+        assert_eq!(detect_file_type("unknown.dat", csv), FileType::Csv);
+    }
+
+    #[test]
+    fn content_hash_deterministic() {
+        let h1 = content_hash(b"hello world");
+        let h2 = content_hash(b"hello world");
+        assert_eq!(h1, h2);
+    }
+}
@@ -0,0 +1,167 @@
+use arrow::array::{ArrayRef, Float64Array, Int64Array, StringArray, BooleanArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::record_batch::RecordBatch;
+use std::collections::BTreeMap;
+use std::sync::Arc;
+
+/// Parse JSON (array of objects or newline-delimited) into Arrow RecordBatches.
+/// Nested objects are flattened: {"a": {"b": 1}} → column "a_b".
+pub fn parse_json(content: &[u8]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
+    let text = std::str::from_utf8(content).map_err(|e| format!("invalid UTF-8: {e}"))?;
+    let text = text.trim();
+
+    // Parse into Vec<Map>
+    let rows: Vec<serde_json::Map<String, serde_json::Value>> = if text.starts_with('[') {
+        // JSON array
+        let arr: Vec<serde_json::Value> = serde_json::from_str(text)
+            .map_err(|e| format!("JSON parse error: {e}"))?;
+        arr.into_iter()
+            .filter_map(|v| v.as_object().cloned())
+            .collect()
+    } else {
+        // Newline-delimited JSON
+        text.lines()
+            .filter(|l| !l.trim().is_empty())
+            .map(|l| {
+                let v: serde_json::Value = serde_json::from_str(l)
+                    .map_err(|e| format!("NDJSON line parse error: {e}"))?;
+                v.as_object().cloned().ok_or_else(|| "NDJSON line is not an object".into())
+            })
+            .collect::<Result<Vec<_>, String>>()?
+    };
+
+    if rows.is_empty() {
+        return Err("JSON has no records".into());
+    }
+
+    // Flatten and collect all columns
+    let mut columns: BTreeMap<String, Vec<serde_json::Value>> = BTreeMap::new();
+    let n_rows = rows.len();
+
+    for (row_idx, row) in rows.iter().enumerate() {
+        let flat = flatten_object(row, "");
+        // Pad existing columns that aren't in this row
+        for key in columns.keys().cloned().collect::<Vec<_>>() {
+            if !flat.contains_key(&key) {
+                columns.get_mut(&key).unwrap().push(serde_json::Value::Null);
+            }
+        }
+        // Add new columns or append values
+        for (key, val) in flat {
+            let col = columns.entry(key).or_insert_with(Vec::new);
+            // Pad with nulls if this column is new and we've seen prior rows
+            while col.len() < row_idx {
+                col.push(serde_json::Value::Null);
+            }
+            col.push(val);
+        }
+    }
+
+    // Pad short columns
+    for col in columns.values_mut() {
+        while col.len() < n_rows {
+            col.push(serde_json::Value::Null);
+        }
+    }
+
+    tracing::info!("parsed JSON: {n_rows} rows × {} columns", columns.len());
+
+    // Infer types and build schema
+    let mut fields = Vec::new();
+    let mut arrays: Vec<ArrayRef> = Vec::new();
+
+    for (name, values) in &columns {
+        let (dt, array) = json_column_to_arrow(values);
+        fields.push(Field::new(name, dt, true));
+        arrays.push(array);
+    }
+
+    let schema = Arc::new(Schema::new(fields));
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    Ok((schema, vec![batch]))
+}
+
+/// Flatten nested JSON object: {"a": {"b": 1}} → {"a_b": 1}
+fn flatten_object(obj: &serde_json::Map<String, serde_json::Value>, prefix: &str) -> BTreeMap<String, serde_json::Value> {
+    let mut result = BTreeMap::new();
+    for (key, val) in obj {
+        let full_key = if prefix.is_empty() { key.clone() } else { format!("{prefix}_{key}") };
+        match val {
+            serde_json::Value::Object(inner) => {
+                result.extend(flatten_object(inner, &full_key));
+            }
+            other => {
+                result.insert(full_key, other.clone());
+            }
+        }
+    }
+    result
+}
+
+/// Convert a column of JSON values to an Arrow array.
+fn json_column_to_arrow(values: &[serde_json::Value]) -> (DataType, ArrayRef) {
+    // Check if all non-null values are the same type
+    let non_null: Vec<&serde_json::Value> = values.iter()
+        .filter(|v| !v.is_null())
+        .collect();
+
+    if non_null.is_empty() {
+        return (DataType::Utf8, Arc::new(StringArray::from(vec![None::<&str>; values.len()])));
+    }
+
+    let all_bool = non_null.iter().all(|v| v.is_boolean());
+    let all_i64 = non_null.iter().all(|v| v.is_i64() || v.is_u64());
+    let all_f64 = non_null.iter().all(|v| v.is_number());
+
+    if all_bool {
+        let arr: Vec<Option<bool>> = values.iter().map(|v| v.as_bool()).collect();
+        (DataType::Boolean, Arc::new(BooleanArray::from(arr)))
+    } else if all_i64 {
+        let arr: Vec<Option<i64>> = values.iter().map(|v| v.as_i64()).collect();
+        (DataType::Int64, Arc::new(Int64Array::from(arr)))
+    } else if all_f64 {
+        let arr: Vec<Option<f64>> = values.iter().map(|v| v.as_f64()).collect();
+        (DataType::Float64, Arc::new(Float64Array::from(arr)))
+    } else {
+        // Default to string
+        let arr: Vec<Option<String>> = values.iter().map(|v| {
+            match v {
+                serde_json::Value::Null => None,
+                serde_json::Value::String(s) => Some(s.clone()),
+                other => Some(other.to_string()),
+            }
+        }).collect();
+        (DataType::Utf8, Arc::new(StringArray::from(arr)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_json_array() {
+        let json = br#"[{"name":"Alice","age":30},{"name":"Bob","age":25}]"#;
+        let (schema, batches) = parse_json(json).unwrap();
+        assert_eq!(batches[0].num_rows(), 2);
+        assert!(schema.field_with_name("age").unwrap().data_type() == &DataType::Int64);
+    }
+
+    #[test]
+    fn parse_ndjson() {
+        let json = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
+        let (_, batches) = parse_json(json).unwrap();
+        assert_eq!(batches[0].num_rows(), 3);
+    }
+
+    #[test]
+    fn flatten_nested() {
+        let json = br#"[{"user":{"name":"Alice","address":{"city":"NYC"}},"score":9.5}]"#;
+        let (schema, _) = parse_json(json).unwrap();
+        let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
+        assert!(names.contains(&"user_name"));
+        assert!(names.contains(&"user_address_city"));
+    }
+}
@@ -0,0 +1,10 @@
+pub mod chatgpt_parser;
+pub mod db_ingest;
+pub mod detect;
+pub mod csv_ingest;
+pub mod json_ingest;
+pub mod pdf_ingest;
+pub mod pipeline;
+pub mod schema_evolution;
+pub mod service;
+pub mod watcher;
@@ -0,0 +1,52 @@
+use arrow::array::{ArrayRef, Int32Array, StringArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::record_batch::RecordBatch;
+use std::sync::Arc;
+
+/// Extract text from a PDF file.
+/// Returns one row per page: (page_number, text_content).
+/// This handles text-based PDFs. Scanned/image PDFs need OCR (not implemented yet).
+pub fn parse_pdf(content: &[u8], source_filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
+    let doc = lopdf::Document::load_mem(content)
+        .map_err(|e| format!("PDF load error: {e}"))?;
+
+    let pages = doc.get_pages();
+    let mut page_numbers: Vec<i32> = Vec::new();
+    let mut page_texts: Vec<String> = Vec::new();
+    let mut sources: Vec<String> = Vec::new();
+
+    for (&page_num, _) in pages.iter() {
+        let text = doc.extract_text(&[page_num]).unwrap_or_default();
+        let text = text.trim().to_string();
+
+        if !text.is_empty() {
+            page_numbers.push(page_num as i32);
+            page_texts.push(text);
+            sources.push(source_filename.to_string());
+        }
+    }
+
+    if page_numbers.is_empty() {
+        // PDF has no extractable text — likely scanned/image
+        return Err("PDF contains no extractable text (may be scanned/image — OCR not yet supported)".into());
+    }
+
+    tracing::info!("extracted {} pages with text from PDF '{}'", page_numbers.len(), source_filename);
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("source_file", DataType::Utf8, false),
+        Field::new("page_number", DataType::Int32, false),
+        Field::new("text_content", DataType::Utf8, false),
+    ]));
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(StringArray::from(sources)),
+        Arc::new(Int32Array::from(page_numbers)),
+        Arc::new(StringArray::from(page_texts)),
+    ];
+
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    Ok((schema, vec![batch]))
+}
@@ -0,0 +1,204 @@
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use bytes::Bytes;
+use object_store::ObjectStore;
+use std::sync::Arc;
+
+use catalogd::registry::Registry;
+use shared::arrow_helpers::{fingerprint_schema, record_batch_to_parquet};
+use shared::types::{ObjectRef, SchemaFingerprint};
+use storaged::ops;
+
+use crate::detect::{FileType, content_hash, detect_file_type};
+use crate::csv_ingest;
+use crate::json_ingest;
+use crate::pdf_ingest;
+
+/// Result of an ingest operation.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct IngestResult {
+    pub dataset_name: String,
+    pub file_type: String,
+    pub rows: usize,
+    pub columns: usize,
+    pub storage_key: String,
+    pub content_hash: String,
+    pub schema_fingerprint: String,
+    pub deduplicated: bool,
+}
+
+/// Full ingest pipeline: detect → parse → dedup → store → register.
+pub async fn ingest_file(
+    filename: &str,
+    content: &[u8],
+    dataset_name: Option<&str>,
+    store: &Arc<dyn ObjectStore>,
+    registry: &Registry,
+) -> Result<IngestResult, String> {
+    // 1. Detect file type
+    let file_type = detect_file_type(filename, content);
+    tracing::info!("ingesting '{}' as {:?} ({} bytes)", filename, file_type, content.len());
+
+    // 2. Parse into Arrow
+    let (schema, batches) = match file_type {
+        FileType::Csv => csv_ingest::parse_csv(content)?,
+        FileType::Json | FileType::NdJson => json_ingest::parse_json(content)?,
+        FileType::Pdf => {
+            // Try ChatGPT parser first, fall back to generic PDF
+            match crate::chatgpt_parser::parse_chatgpt_pdf(content, filename) {
+                Ok(result) => result,
+                Err(_) => pdf_ingest::parse_pdf(content, filename)?,
+            }
+        }
+        FileType::Text => parse_text_file(content, filename)?,
+        FileType::Unknown => return Err(format!("unknown file type for '{filename}'")),
+    };
+
+    let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+    let total_cols = schema.fields().len();
+
+    if total_rows == 0 {
+        return Err("file contains no data".into());
+    }
+
+    // 3. Content hash for dedup
+    let hash = content_hash(content);
+
+    // 4. Check if already ingested (same content hash)
+    let existing = registry.list().await;
+    if existing.iter().any(|d| d.schema_fingerprint.0 == hash) {
+        tracing::info!("file already ingested (hash: {}), skipping", &hash[..12]);
+        return Ok(IngestResult {
+            dataset_name: dataset_name.unwrap_or(filename).to_string(),
+            file_type: format!("{:?}", file_type),
+            rows: total_rows,
+            columns: total_cols,
+            storage_key: String::new(),
+            content_hash: hash,
+            schema_fingerprint: String::new(),
+            deduplicated: true,
+        });
+    }
+
+    // 5. Convert to Parquet
+    let mut all_parquet = Vec::new();
+    for batch in &batches {
+        let pq = record_batch_to_parquet(batch)?;
+        all_parquet.extend_from_slice(&pq);
+    }
+    let parquet_bytes = Bytes::from(all_parquet);
+    let parquet_size = parquet_bytes.len() as u64;
+
+    // 6. Store in object storage
+    let name = dataset_name.unwrap_or_else(|| {
+        filename.rsplit('/').next().unwrap_or(filename)
+            .rsplit('.').last().unwrap_or(filename)
+    });
+    let safe_name = sanitize_dataset_name(name);
+    let storage_key = format!("datasets/{}.parquet", safe_name);
+
+    ops::put(store, &storage_key, parquet_bytes).await?;
+    tracing::info!("stored {} as {} ({} bytes)", filename, storage_key, parquet_size);
+
+    // 7. Register in catalog with rich metadata
+    let schema_fp = fingerprint_schema(&schema);
+    let now = chrono::Utc::now();
+    let obj_ref = ObjectRef {
+        bucket: "data".to_string(),
+        key: storage_key.clone(),
+        size_bytes: parquet_size,
+        created_at: now,
+    };
+
+    // Register base dataset
+    registry.register(
+        safe_name.clone(),
+        SchemaFingerprint(hash.clone()),
+        vec![obj_ref],
+    ).await?;
+
+    // Auto-populate metadata: PII detection, column info, lineage, row count
+    let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
+    let dataset_sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
+
+    let column_meta: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
+        let sens = shared::pii::detect_sensitivity(f.name());
+        shared::types::ColumnMeta {
+            name: f.name().clone(),
+            data_type: f.data_type().to_string(),
+            sensitivity: sens.clone(),
+            description: String::new(),
+            is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
+        }
+    }).collect();
+
+    let lineage = shared::types::Lineage {
+        source_system: format!("{:?}", file_type).to_lowercase(),
+        source_file: filename.to_string(),
+        ingest_job: format!("ingest-{}", now.timestamp_millis()),
+        ingest_timestamp: now,
+        parent_datasets: vec![],
+    };
+
+    let pii_count = column_meta.iter().filter(|c| c.is_pii).count();
+    if pii_count > 0 {
+        tracing::info!("auto-detected {} PII columns in '{}'", pii_count, safe_name);
+    }
+
+    let _ = registry.update_metadata(&safe_name, catalogd::registry::MetadataUpdate {
+        sensitivity: dataset_sensitivity,
+        columns: Some(column_meta),
+        lineage: Some(lineage),
+        row_count: Some(total_rows as u64),
+        ..Default::default()
+    }).await;
+
+    Ok(IngestResult {
+        dataset_name: safe_name,
+        file_type: format!("{:?}", file_type),
+        rows: total_rows,
+        columns: total_cols,
+        storage_key,
+        content_hash: hash,
+        schema_fingerprint: schema_fp.0,
+        deduplicated: false,
+    })
+}
+
+/// Parse plain text / SMS logs into a simple table.
+fn parse_text_file(content: &[u8], filename: &str) -> Result<(SchemaRef, Vec<RecordBatch>), String> {
+    let text = String::from_utf8_lossy(content);
+    let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
+
+    if lines.is_empty() {
+        return Err("text file is empty".into());
+    }
+
+    let schema = Arc::new(arrow::datatypes::Schema::new(vec![
+        arrow::datatypes::Field::new("source_file", arrow::datatypes::DataType::Utf8, false),
+        arrow::datatypes::Field::new("line_number", arrow::datatypes::DataType::Int32, false),
+        arrow::datatypes::Field::new("text", arrow::datatypes::DataType::Utf8, false),
+    ]));
+
+    let sources: Vec<&str> = vec![filename; lines.len()];
+    let line_nums: Vec<i32> = (1..=lines.len() as i32).collect();
+
+    let arrays: Vec<arrow::array::ArrayRef> = vec![
+        Arc::new(arrow::array::StringArray::from(sources)),
+        Arc::new(arrow::array::Int32Array::from(line_nums)),
+        Arc::new(arrow::array::StringArray::from(lines)),
+    ];
+
+    let batch = RecordBatch::try_new(schema.clone(), arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    Ok((schema, vec![batch]))
+}
+
+fn sanitize_dataset_name(name: &str) -> String {
+    let clean: String = name.chars()
+        .map(|c| if c.is_alphanumeric() || c == '_' { c.to_ascii_lowercase() } else { '_' })
+        .collect();
+    let trimmed = clean.trim_matches('_').to_string();
+    if trimmed.is_empty() { "unnamed_dataset".to_string() } else { trimmed }
+}
@@ -0,0 +1,279 @@
+/// Schema evolution: detect changes between ingests and suggest migration rules.
+/// When a source changes format (columns renamed, added, removed, type changed),
+/// the system detects the diff and can auto-map using AI or heuristic matching.
+
+use arrow::datatypes::{DataType, Schema, SchemaRef};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+/// A detected change between two schema versions.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SchemaChange {
+    pub change_type: ChangeType,
+    pub column: String,
+    pub detail: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum ChangeType {
+    Added,         // new column in incoming schema
+    Removed,       // column in old schema not in new
+    TypeChanged,   // same name, different type
+    Renamed,       // likely renamed (fuzzy match)
+}
+
+/// A migration rule — how to map old schema to new.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct MigrationRule {
+    pub source_column: String,
+    pub target_column: String,
+    pub transform: String,         // "direct", "cast", "concat", "split", "drop"
+    pub confidence: f32,           // 0.0 - 1.0
+    pub auto_generated: bool,
+}
+
+/// Compare two schemas and detect changes.
+pub fn diff_schemas(old: &SchemaRef, new: &SchemaRef) -> Vec<SchemaChange> {
+    let mut changes = Vec::new();
+
+    let old_fields: HashMap<&str, &DataType> = old.fields().iter()
+        .map(|f| (f.name().as_str(), f.data_type()))
+        .collect();
+
+    let new_fields: HashMap<&str, &DataType> = new.fields().iter()
+        .map(|f| (f.name().as_str(), f.data_type()))
+        .collect();
+
+    // Check for removed columns
+    for (name, _dt) in &old_fields {
+        if !new_fields.contains_key(name) {
+            // Check if it was renamed (fuzzy match)
+            let possible_rename = find_similar_column(name, &new_fields, &old_fields);
+            if let Some((new_name, confidence)) = possible_rename {
+                changes.push(SchemaChange {
+                    change_type: ChangeType::Renamed,
+                    column: name.to_string(),
+                    detail: format!("likely renamed to '{}' (confidence: {:.0}%)", new_name, confidence * 100.0),
+                });
+            } else {
+                changes.push(SchemaChange {
+                    change_type: ChangeType::Removed,
+                    column: name.to_string(),
+                    detail: "column no longer present in incoming data".into(),
+                });
+            }
+        }
+    }
+
+    // Check for added or type-changed columns
+    for (name, new_dt) in &new_fields {
+        match old_fields.get(name) {
+            None => {
+                // Skip if we already flagged this as a rename target
+                let is_rename_target = changes.iter().any(|c| {
+                    c.change_type == ChangeType::Renamed && c.detail.contains(name)
+                });
+                if !is_rename_target {
+                    changes.push(SchemaChange {
+                        change_type: ChangeType::Added,
+                        column: name.to_string(),
+                        detail: format!("new column (type: {})", new_dt),
+                    });
+                }
+            }
+            Some(old_dt) => {
+                if old_dt != new_dt {
+                    changes.push(SchemaChange {
+                        change_type: ChangeType::TypeChanged,
+                        column: name.to_string(),
+                        detail: format!("{} → {}", old_dt, new_dt),
+                    });
+                }
+            }
+        }
+    }
+
+    changes
+}
+
+/// Generate migration rules from detected changes.
+pub fn generate_rules(changes: &[SchemaChange]) -> Vec<MigrationRule> {
+    changes.iter().filter_map(|change| {
+        match change.change_type {
+            ChangeType::Renamed => {
+                // Extract the new name from the detail string
+                let new_name = change.detail
+                    .split('\'')
+                    .nth(1)
+                    .unwrap_or(&change.column);
+                Some(MigrationRule {
+                    source_column: change.column.clone(),
+                    target_column: new_name.to_string(),
+                    transform: "direct".into(),
+                    confidence: 0.8,
+                    auto_generated: true,
+                })
+            }
+            ChangeType::TypeChanged => {
+                Some(MigrationRule {
+                    source_column: change.column.clone(),
+                    target_column: change.column.clone(),
+                    transform: "cast".into(),
+                    confidence: 0.9,
+                    auto_generated: true,
+                })
+            }
+            ChangeType::Added => {
+                Some(MigrationRule {
+                    source_column: String::new(),
+                    target_column: change.column.clone(),
+                    transform: "new_column".into(),
+                    confidence: 1.0,
+                    auto_generated: true,
+                })
+            }
+            ChangeType::Removed => {
+                Some(MigrationRule {
+                    source_column: change.column.clone(),
+                    target_column: String::new(),
+                    transform: "drop".into(),
+                    confidence: 0.7,
+                    auto_generated: true,
+                })
+            }
+        }
+    }).collect()
+}
+
+/// Build an AI prompt to suggest migration rules for complex cases.
+pub fn build_migration_prompt(
+    old_schema: &SchemaRef,
+    new_schema: &SchemaRef,
+    changes: &[SchemaChange],
+) -> String {
+    let old_cols: Vec<String> = old_schema.fields().iter()
+        .map(|f| format!("  {} ({})", f.name(), f.data_type()))
+        .collect();
+    let new_cols: Vec<String> = new_schema.fields().iter()
+        .map(|f| format!("  {} ({})", f.name(), f.data_type()))
+        .collect();
+    let change_list: Vec<String> = changes.iter()
+        .map(|c| format!("  [{:?}] {} — {}", c.change_type, c.column, c.detail))
+        .collect();
+
+    format!(
+        "You are a data migration expert. A data source has changed its schema.\n\n\
+        OLD SCHEMA:\n{}\n\n\
+        NEW SCHEMA:\n{}\n\n\
+        DETECTED CHANGES:\n{}\n\n\
+        For each change, suggest a migration rule as JSON:\n\
+        {{\"source_column\": \"old_name\", \"target_column\": \"new_name\", \"transform\": \"direct|cast|concat|split|drop\", \"confidence\": 0.0-1.0}}\n\n\
+        Output ONLY a JSON array of rules, nothing else.",
+        old_cols.join("\n"),
+        new_cols.join("\n"),
+        change_list.join("\n"),
+    )
+}
+
+/// Fuzzy match: find a column in new schema that's similar to a removed old column.
+fn find_similar_column<'a>(
+    old_name: &str,
+    new_fields: &HashMap<&'a str, &DataType>,
+    old_fields: &HashMap<&str, &DataType>,
+) -> Option<(String, f32)> {
+    let old_lower = old_name.to_lowercase();
+    let old_parts: Vec<&str> = old_lower.split('_').collect();
+
+    let mut best_match: Option<(String, f32)> = None;
+
+    for (new_name, _) in new_fields {
+        // Skip columns that already exist in old schema (not renamed)
+        if old_fields.contains_key(new_name) { continue; }
+
+        let new_lower = new_name.to_lowercase();
+        let new_parts: Vec<&str> = new_lower.split('_').collect();
+
+        // Exact substring match
+        if old_lower.contains(&new_lower) || new_lower.contains(&old_lower) {
+            let confidence = 0.85;
+            if best_match.as_ref().map_or(true, |(_, c)| confidence > *c) {
+                best_match = Some((new_name.to_string(), confidence));
+            }
+            continue;
+        }
+
+        // Shared word parts (e.g., "first_name" → "fname", "full_name")
+        let shared: usize = old_parts.iter()
+            .filter(|p| new_parts.iter().any(|np| np.contains(*p) || p.contains(np)))
+            .count();
+
+        if shared > 0 {
+            let confidence = shared as f32 / old_parts.len().max(new_parts.len()) as f32 * 0.75;
+            if confidence >= 0.3 && best_match.as_ref().map_or(true, |(_, c)| confidence > *c) {
+                best_match = Some((new_name.to_string(), confidence));
+            }
+        }
+    }
+
+    best_match
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow::datatypes::Field;
+
+    fn schema(fields: Vec<(&str, DataType)>) -> SchemaRef {
+        Arc::new(Schema::new(
+            fields.into_iter().map(|(n, t)| Field::new(n, t, true)).collect::<Vec<_>>()
+        ))
+    }
+
+    #[test]
+    fn detect_added_column() {
+        let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
+        let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("email", DataType::Utf8)]);
+        let changes = diff_schemas(&old, &new);
+        assert_eq!(changes.len(), 1);
+        assert_eq!(changes[0].change_type, ChangeType::Added);
+        assert_eq!(changes[0].column, "email");
+    }
+
+    #[test]
+    fn detect_removed_column() {
+        let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("age", DataType::Int64)]);
+        let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
+        let changes = diff_schemas(&old, &new);
+        assert!(changes.iter().any(|c| c.column == "age" && c.change_type == ChangeType::Removed));
+    }
+
+    #[test]
+    fn detect_type_change() {
+        let old = schema(vec![("id", DataType::Int64), ("score", DataType::Int64)]);
+        let new = schema(vec![("id", DataType::Int64), ("score", DataType::Float64)]);
+        let changes = diff_schemas(&old, &new);
+        assert_eq!(changes.len(), 1);
+        assert_eq!(changes[0].change_type, ChangeType::TypeChanged);
+    }
+
+    #[test]
+    fn detect_rename() {
+        let old = schema(vec![("first_name", DataType::Utf8), ("last_name", DataType::Utf8)]);
+        let new = schema(vec![("full_name", DataType::Utf8)]);
+        let changes = diff_schemas(&old, &new);
+        // Should detect first_name → full_name as a potential rename
+        assert!(changes.iter().any(|c| c.change_type == ChangeType::Renamed));
+    }
+
+    #[test]
+    fn generate_rules_from_changes() {
+        let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
+        let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("email", DataType::Utf8)]);
+        let changes = diff_schemas(&old, &new);
+        let rules = generate_rules(&changes);
+        assert_eq!(rules.len(), 1);
+        assert_eq!(rules[0].transform, "new_column");
+    }
+}
@@ -0,0 +1,190 @@
+use axum::{
+    Json, Router,
+    extract::{Multipart, Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use bytes::Bytes;
+use object_store::ObjectStore;
+use serde::Deserialize;
+use std::sync::Arc;
+
+use catalogd::registry::Registry;
+use crate::{db_ingest, pipeline};
+use shared::arrow_helpers::record_batch_to_parquet;
+use shared::types::{ObjectRef, SchemaFingerprint};
+use storaged::ops;
+
+#[derive(Clone)]
+pub struct IngestState {
+    pub store: Arc<dyn ObjectStore>,
+    pub registry: Registry,
+}
+
+pub fn router(state: IngestState) -> Router {
+    Router::new()
+        .route("/health", get(health))
+        .route("/file", post(ingest_file))
+        .route("/postgres/tables", post(list_pg_tables))
+        .route("/postgres/import", post(import_pg_table))
+        .with_state(state)
+}
+
+async fn health() -> &'static str {
+    "ingestd ok"
+}
+
+#[derive(Deserialize)]
+struct IngestQuery {
+    name: Option<String>,
+}
+
+async fn ingest_file(
+    State(state): State<IngestState>,
+    Query(query): Query<IngestQuery>,
+    mut multipart: Multipart,
+) -> impl IntoResponse {
+    let field = match multipart.next_field().await {
+        Ok(Some(f)) => f,
+        Ok(None) => return Err((StatusCode::BAD_REQUEST, "no file uploaded".to_string())),
+        Err(e) => return Err((StatusCode::BAD_REQUEST, format!("multipart error: {e}"))),
+    };
+
+    let filename = field.file_name().unwrap_or("unknown").to_string();
+    let content = field.bytes().await
+        .map_err(|e| (StatusCode::BAD_REQUEST, format!("read error: {e}")))?;
+
+    tracing::info!("received file '{}' ({} bytes) for ingest", filename, content.len());
+
+    let dataset_name = query.name.as_deref();
+
+    match pipeline::ingest_file(&filename, &content, dataset_name, &state.store, &state.registry).await {
+        Ok(result) => {
+            if result.deduplicated {
+                Ok((StatusCode::OK, Json(result)))
+            } else {
+                Ok((StatusCode::CREATED, Json(result)))
+            }
+        }
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+// --- PostgreSQL Import ---
+
+/// List tables in a PostgreSQL database.
+async fn list_pg_tables(
+    Json(config): Json<db_ingest::DbConfig>,
+) -> impl IntoResponse {
+    match db_ingest::list_postgres_tables(&config).await {
+        Ok(tables) => Ok(Json(tables)),
+        Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct PgImportRequest {
+    #[serde(flatten)]
+    config: db_ingest::DbConfig,
+    table: String,
+    /// Override dataset name (defaults to table name)
+    dataset_name: Option<String>,
+    /// Max rows to import (None = all)
+    limit: Option<usize>,
+}
+
+/// Import a PostgreSQL table into the lakehouse.
+async fn import_pg_table(
+    State(state): State<IngestState>,
+    Json(req): Json<PgImportRequest>,
+) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
+    tracing::info!("importing postgres table '{}' from {}:{}/{}",
+        req.table, req.config.host, req.config.port, req.config.database);
+
+    // Import from Postgres
+    let (schema, batches, db_result) = db_ingest::import_postgres_table(
+        &req.config, &req.table, req.limit,
+    ).await.map_err(|e| (StatusCode::BAD_GATEWAY, e))?;
+
+    if batches.is_empty() || db_result.rows == 0 {
+        return Ok((StatusCode::OK, Json(serde_json::json!({
+            "table": req.table,
+            "rows": 0,
+            "message": "table is empty",
+        }))));
+    }
+
+    // Convert to Parquet
+    let mut all_parquet = Vec::new();
+    for batch in &batches {
+        let pq = record_batch_to_parquet(batch)
+            .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
+        all_parquet.extend_from_slice(&pq);
+    }
+
+    let dataset_name = req.dataset_name.unwrap_or_else(|| req.table.clone());
+    let storage_key = format!("datasets/{}.parquet", dataset_name);
+    let parquet_size = all_parquet.len() as u64;
+
+    // Store
+    ops::put(&state.store, &storage_key, Bytes::from(all_parquet))
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
+
+    // Register
+    let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
+    let now = chrono::Utc::now();
+    state.registry.register(
+        dataset_name.clone(),
+        SchemaFingerprint(schema_fp.0),
+        vec![ObjectRef {
+            bucket: "data".to_string(),
+            key: storage_key.clone(),
+            size_bytes: parquet_size,
+            created_at: now,
+        }],
+    ).await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
+
+    // Auto-populate metadata (PII detection, lineage)
+    let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
+    let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
+    let columns: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
+        let sens = shared::pii::detect_sensitivity(f.name());
+        shared::types::ColumnMeta {
+            name: f.name().clone(),
+            data_type: f.data_type().to_string(),
+            sensitivity: sens.clone(),
+            description: String::new(),
+            is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
+        }
+    }).collect();
+
+    let lineage = shared::types::Lineage {
+        source_system: "postgresql".to_string(),
+        source_file: format!("{}:{}/{}.{}", req.config.host, req.config.port, req.config.database, req.table),
+        ingest_job: format!("pg-import-{}", now.timestamp_millis()),
+        ingest_timestamp: now,
+        parent_datasets: vec![],
+    };
+
+    let _ = state.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
+        sensitivity,
+        columns: Some(columns),
+        lineage: Some(lineage),
+        row_count: Some(db_result.rows as u64),
+        ..Default::default()
+    }).await;
+
+    tracing::info!("imported '{}' from postgres: {} rows → {}", dataset_name, db_result.rows, storage_key);
+
+    Ok((StatusCode::CREATED, Json(serde_json::json!({
+        "dataset_name": dataset_name,
+        "table": req.table,
+        "rows": db_result.rows,
+        "columns": db_result.columns,
+        "schema": db_result.schema_detected,
+        "storage_key": storage_key,
+        "size_bytes": parquet_size,
+    }))))
+}
@@ -0,0 +1,122 @@
+/// File watcher: monitors a directory for new files and auto-ingests them.
+/// Drop a CSV, JSON, PDF, or text file → auto-detected → Parquet → queryable.
+/// Processed files are moved to a "processed/" subdirectory.
+
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::time;
+
+use catalogd::registry::Registry;
+use object_store::ObjectStore;
+
+use crate::pipeline;
+
+/// Watch config.
+pub struct WatchConfig {
+    pub watch_dir: PathBuf,
+    pub processed_dir: PathBuf,
+    pub failed_dir: PathBuf,
+    pub poll_interval: Duration,
+}
+
+impl Default for WatchConfig {
+    fn default() -> Self {
+        Self {
+            watch_dir: PathBuf::from("./inbox"),
+            processed_dir: PathBuf::from("./inbox/processed"),
+            failed_dir: PathBuf::from("./inbox/failed"),
+            poll_interval: Duration::from_secs(10),
+        }
+    }
+}
+
+/// Start the file watcher as a background task.
+pub fn spawn_watcher(
+    config: WatchConfig,
+    store: Arc<dyn ObjectStore>,
+    registry: Registry,
+) -> tokio::task::JoinHandle<()> {
+    // Create directories
+    std::fs::create_dir_all(&config.watch_dir).ok();
+    std::fs::create_dir_all(&config.processed_dir).ok();
+    std::fs::create_dir_all(&config.failed_dir).ok();
+
+    tracing::info!("file watcher started: watching {}", config.watch_dir.display());
+
+    tokio::spawn(async move {
+        let mut interval = time::interval(config.poll_interval);
+        loop {
+            interval.tick().await;
+            if let Err(e) = process_inbox(&config, &store, &registry).await {
+                tracing::error!("watcher error: {e}");
+            }
+        }
+    })
+}
+
+/// Scan inbox directory and process any new files.
+async fn process_inbox(
+    config: &WatchConfig,
+    store: &Arc<dyn ObjectStore>,
+    registry: &Registry,
+) -> Result<(), String> {
+    let entries = std::fs::read_dir(&config.watch_dir)
+        .map_err(|e| format!("can't read inbox: {e}"))?;
+
+    for entry in entries {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+
+        let path = entry.path();
+
+        // Skip directories and hidden files
+        if path.is_dir() || path.file_name().map_or(true, |n| n.to_string_lossy().starts_with('.')) {
+            continue;
+        }
+
+        let filename = path.file_name().unwrap().to_string_lossy().to_string();
+        tracing::info!("watcher: found new file '{}'", filename);
+
+        // Read file
+        let content = match std::fs::read(&path) {
+            Ok(c) => c,
+            Err(e) => {
+                tracing::error!("watcher: can't read '{}': {e}", filename);
+                move_file(&path, &config.failed_dir.join(&filename));
+                continue;
+            }
+        };
+
+        // Ingest
+        match pipeline::ingest_file(&filename, &content, None, store, registry).await {
+            Ok(result) => {
+                if result.deduplicated {
+                    tracing::info!("watcher: '{}' already ingested, skipping", filename);
+                } else {
+                    tracing::info!("watcher: ingested '{}' → {} ({} rows, {} cols)",
+                        filename, result.dataset_name, result.rows, result.columns);
+                }
+                // Move to processed
+                move_file(&path, &config.processed_dir.join(&filename));
+            }
+            Err(e) => {
+                tracing::error!("watcher: failed to ingest '{}': {e}", filename);
+                move_file(&path, &config.failed_dir.join(&filename));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn move_file(from: &Path, to: &Path) {
+    if let Err(e) = std::fs::rename(from, to) {
+        // rename fails across filesystems, try copy+delete
+        if let Err(e2) = std::fs::copy(from, to).and_then(|_| std::fs::remove_file(from)) {
+            tracing::warn!("can't move {} → {}: {e}, copy failed: {e2}", from.display(), to.display());
+        }
+    }
+}
@@ -0,0 +1,18 @@
+[package]
+name = "journald"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+shared = { path = "../shared" }
+storaged = { path = "../storaged" }
+tokio = { workspace = true }
+axum = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tracing = { workspace = true }
+arrow = { workspace = true }
+parquet = { workspace = true }
+bytes = { workspace = true }
+chrono = { workspace = true }
+object_store = { workspace = true }
@@ -0,0 +1,322 @@
+/// Append-only event journal.
+/// Every mutation to any dataset is recorded here as an immutable event.
+/// Events are never modified or deleted — this IS the audit trail.
+///
+/// Storage: events buffer in memory, flush to Parquet periodically.
+/// Query: load Parquet files, filter by entity/field/actor/time.
+
+use arrow::array::{ArrayRef, RecordBatch, StringArray, UInt64Array};
+use arrow::datatypes::{DataType, Field, Schema};
+use chrono::{DateTime, Utc};
+use object_store::ObjectStore;
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use shared::arrow_helpers::record_batch_to_parquet;
+use storaged::ops;
+
+/// A single mutation event. Immutable once created.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Event {
+    pub event_id: String,
+    pub timestamp: DateTime<Utc>,
+    pub entity_type: String,    // "candidate", "placement", "client", etc.
+    pub entity_id: String,      // "CAND-00342"
+    pub field: String,           // "phone", "*" for insert/delete
+    pub action: String,          // "insert", "update", "delete"
+    pub old_value: String,       // "" for inserts
+    pub new_value: String,       // "" for deletes
+    pub actor: String,           // "Sarah", "ingest_pipeline", "auto_matcher"
+    pub source: String,          // "api", "ingest", "workspace", "agent"
+    pub workspace_id: String,    // "" if not workspace-scoped
+}
+
+/// Arrow schema for journal Parquet files.
+fn journal_schema() -> Arc<Schema> {
+    Arc::new(Schema::new(vec![
+        Field::new("event_id", DataType::Utf8, false),
+        Field::new("timestamp", DataType::Utf8, false),
+        Field::new("entity_type", DataType::Utf8, false),
+        Field::new("entity_id", DataType::Utf8, false),
+        Field::new("field", DataType::Utf8, false),
+        Field::new("action", DataType::Utf8, false),
+        Field::new("old_value", DataType::Utf8, true),
+        Field::new("new_value", DataType::Utf8, true),
+        Field::new("actor", DataType::Utf8, false),
+        Field::new("source", DataType::Utf8, false),
+        Field::new("workspace_id", DataType::Utf8, true),
+    ]))
+}
+
+/// The event journal — buffers events in memory, flushes to Parquet.
+#[derive(Clone)]
+pub struct Journal {
+    buffer: Arc<RwLock<Vec<Event>>>,
+    store: Arc<dyn ObjectStore>,
+    flush_threshold: usize,
+    event_counter: Arc<std::sync::atomic::AtomicU64>,
+}
+
+impl Journal {
+    pub fn new(store: Arc<dyn ObjectStore>, flush_threshold: usize) -> Self {
+        Self {
+            buffer: Arc::new(RwLock::new(Vec::new())),
+            store,
+            flush_threshold,
+            event_counter: Arc::new(std::sync::atomic::AtomicU64::new(0)),
+        }
+    }
+
+    /// Record a single event. Triggers flush if buffer exceeds threshold.
+    pub async fn record(&self, mut event: Event) -> Result<(), String> {
+        let seq = self.event_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        event.event_id = format!("evt-{}-{seq}", event.timestamp.timestamp_millis());
+
+        let should_flush = {
+            let mut buf = self.buffer.write().await;
+            buf.push(event);
+            buf.len() >= self.flush_threshold
+        };
+
+        if should_flush {
+            self.flush().await?;
+        }
+
+        Ok(())
+    }
+
+    /// Record a batch insert (multiple rows added to a dataset).
+    pub async fn record_insert(
+        &self,
+        entity_type: &str,
+        entity_id: &str,
+        actor: &str,
+        source: &str,
+        workspace_id: &str,
+    ) -> Result<(), String> {
+        self.record(Event {
+            event_id: String::new(),
+            timestamp: Utc::now(),
+            entity_type: entity_type.to_string(),
+            entity_id: entity_id.to_string(),
+            field: "*".to_string(),
+            action: "insert".to_string(),
+            old_value: String::new(),
+            new_value: String::new(),
+            actor: actor.to_string(),
+            source: source.to_string(),
+            workspace_id: workspace_id.to_string(),
+        }).await
+    }
+
+    /// Record a field update.
+    pub async fn record_update(
+        &self,
+        entity_type: &str,
+        entity_id: &str,
+        field: &str,
+        old_value: &str,
+        new_value: &str,
+        actor: &str,
+        source: &str,
+        workspace_id: &str,
+    ) -> Result<(), String> {
+        self.record(Event {
+            event_id: String::new(),
+            timestamp: Utc::now(),
+            entity_type: entity_type.to_string(),
+            entity_id: entity_id.to_string(),
+            field: field.to_string(),
+            action: "update".to_string(),
+            old_value: old_value.to_string(),
+            new_value: new_value.to_string(),
+            actor: actor.to_string(),
+            source: source.to_string(),
+            workspace_id: workspace_id.to_string(),
+        }).await
+    }
+
+    /// Record a batch of events from a dataset ingest.
+    pub async fn record_ingest(
+        &self,
+        entity_type: &str,
+        count: usize,
+        actor: &str,
+        source_file: &str,
+    ) -> Result<(), String> {
+        self.record(Event {
+            event_id: String::new(),
+            timestamp: Utc::now(),
+            entity_type: entity_type.to_string(),
+            entity_id: format!("batch:{count}"),
+            field: "*".to_string(),
+            action: "ingest".to_string(),
+            old_value: String::new(),
+            new_value: format!("{count} rows from {source_file}"),
+            actor: actor.to_string(),
+            source: "ingest".to_string(),
+            workspace_id: String::new(),
+        }).await
+    }
+
+    /// Flush buffer to Parquet. Called automatically at threshold or manually.
+    pub async fn flush(&self) -> Result<usize, String> {
+        let events = {
+            let mut buf = self.buffer.write().await;
+            if buf.is_empty() { return Ok(0); }
+            std::mem::take(&mut *buf)
+        };
+
+        let n = events.len();
+        let batch = events_to_batch(&events)?;
+        let parquet = record_batch_to_parquet(&batch)?;
+
+        let ts = Utc::now().format("%Y%m%d_%H%M%S_%3f");
+        let key = format!("journal/{ts}.parquet");
+        ops::put(&self.store, &key, parquet).await?;
+
+        tracing::info!("journal: flushed {n} events to {key}");
+        Ok(n)
+    }
+
+    /// Query journal: get all events for a specific entity.
+    pub async fn get_entity_history(&self, entity_id: &str) -> Result<Vec<Event>, String> {
+        let mut all_events = Vec::new();
+
+        // Check in-memory buffer first
+        {
+            let buf = self.buffer.read().await;
+            for e in buf.iter() {
+                if e.entity_id == entity_id {
+                    all_events.push(e.clone());
+                }
+            }
+        }
+
+        // Load from persisted journal files
+        let keys = ops::list(&self.store, Some("journal/")).await?;
+        for key in &keys {
+            if !key.ends_with(".parquet") { continue; }
+            let data = ops::get(&self.store, key).await?;
+            let (_, batches) = shared::arrow_helpers::parquet_to_record_batches(&data)?;
+            for batch in &batches {
+                let events = batch_to_events(batch)?;
+                for e in events {
+                    if e.entity_id == entity_id {
+                        all_events.push(e);
+                    }
+                }
+            }
+        }
+
+        all_events.sort_by_key(|e| e.timestamp);
+        Ok(all_events)
+    }
+
+    /// Query journal: get recent events (last N).
+    pub async fn get_recent(&self, limit: usize) -> Result<Vec<Event>, String> {
+        let mut all_events = Vec::new();
+
+        // Buffer
+        {
+            let buf = self.buffer.read().await;
+            all_events.extend(buf.iter().cloned());
+        }
+
+        // Persisted (load most recent files first)
+        let mut keys = ops::list(&self.store, Some("journal/")).await?;
+        keys.sort();
+        keys.reverse();
+
+        for key in &keys {
+            if !key.ends_with(".parquet") { continue; }
+            if all_events.len() >= limit { break; }
+            let data = ops::get(&self.store, key).await?;
+            let (_, batches) = shared::arrow_helpers::parquet_to_record_batches(&data)?;
+            for batch in &batches {
+                all_events.extend(batch_to_events(batch)?);
+            }
+        }
+
+        all_events.sort_by_key(|e| e.timestamp);
+        all_events.reverse();
+        all_events.truncate(limit);
+        Ok(all_events)
+    }
+
+    /// Get buffer stats.
+    pub async fn stats(&self) -> JournalStats {
+        let buf = self.buffer.read().await;
+        let keys = ops::list(&self.store, Some("journal/")).await.unwrap_or_default();
+        JournalStats {
+            buffer_events: buf.len(),
+            flush_threshold: self.flush_threshold,
+            persisted_files: keys.len(),
+            total_events_created: self.event_counter.load(std::sync::atomic::Ordering::Relaxed),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct JournalStats {
+    pub buffer_events: usize,
+    pub flush_threshold: usize,
+    pub persisted_files: usize,
+    pub total_events_created: u64,
+}
+
+fn events_to_batch(events: &[Event]) -> Result<RecordBatch, String> {
+    let schema = journal_schema();
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(StringArray::from(events.iter().map(|e| e.event_id.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.timestamp.to_rfc3339()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.entity_type.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.entity_id.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.field.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.action.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.old_value.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.new_value.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.actor.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.source.as_str()).collect::<Vec<_>>())),
+        Arc::new(StringArray::from(events.iter().map(|e| e.workspace_id.as_str()).collect::<Vec<_>>())),
+    ];
+    RecordBatch::try_new(schema, arrays).map_err(|e| format!("batch error: {e}"))
+}
+
+fn batch_to_events(batch: &RecordBatch) -> Result<Vec<Event>, String> {
+    let get_col = |i: usize| -> Result<&StringArray, String> {
+        batch.column(i).as_any().downcast_ref::<StringArray>()
+            .ok_or_else(|| format!("column {i} not string"))
+    };
+
+    let event_ids = get_col(0)?;
+    let timestamps = get_col(1)?;
+    let entity_types = get_col(2)?;
+    let entity_ids = get_col(3)?;
+    let fields = get_col(4)?;
+    let actions = get_col(5)?;
+    let old_values = get_col(6)?;
+    let new_values = get_col(7)?;
+    let actors = get_col(8)?;
+    let sources = get_col(9)?;
+    let workspace_ids = get_col(10)?;
+
+    let mut events = Vec::with_capacity(batch.num_rows());
+    for i in 0..batch.num_rows() {
+        events.push(Event {
+            event_id: event_ids.value(i).to_string(),
+            timestamp: timestamps.value(i).parse().unwrap_or_else(|_| Utc::now()),
+            entity_type: entity_types.value(i).to_string(),
+            entity_id: entity_ids.value(i).to_string(),
+            field: fields.value(i).to_string(),
+            action: actions.value(i).to_string(),
+            old_value: old_values.value(i).to_string(),
+            new_value: new_values.value(i).to_string(),
+            actor: actors.value(i).to_string(),
+            source: sources.value(i).to_string(),
+            workspace_id: workspace_ids.value(i).to_string(),
+        });
+    }
+    Ok(events)
+}
@@ -0,0 +1,2 @@
+pub mod journal;
+pub mod service;
@@ -0,0 +1,131 @@
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use serde::Deserialize;
+
+use crate::journal::{Event, Journal};
+
+pub fn router(journal: Journal) -> Router {
+    Router::new()
+        .route("/health", get(health))
+        .route("/event", post(record_event))
+        .route("/update", post(record_update))
+        .route("/flush", post(flush))
+        .route("/history/{entity_id}", get(entity_history))
+        .route("/recent", get(recent_events))
+        .route("/stats", get(stats))
+        .with_state(journal)
+}
+
+async fn health() -> &'static str {
+    "journald ok"
+}
+
+// --- Record events ---
+
+#[derive(Deserialize)]
+struct RecordEventRequest {
+    entity_type: String,
+    entity_id: String,
+    action: String,
+    field: Option<String>,
+    old_value: Option<String>,
+    new_value: Option<String>,
+    actor: String,
+    source: String,
+    workspace_id: Option<String>,
+}
+
+async fn record_event(
+    State(journal): State<Journal>,
+    Json(req): Json<RecordEventRequest>,
+) -> impl IntoResponse {
+    let event = Event {
+        event_id: String::new(), // assigned by journal
+        timestamp: chrono::Utc::now(),
+        entity_type: req.entity_type,
+        entity_id: req.entity_id,
+        field: req.field.unwrap_or_else(|| "*".to_string()),
+        action: req.action,
+        old_value: req.old_value.unwrap_or_default(),
+        new_value: req.new_value.unwrap_or_default(),
+        actor: req.actor,
+        source: req.source,
+        workspace_id: req.workspace_id.unwrap_or_default(),
+    };
+
+    match journal.record(event).await {
+        Ok(()) => Ok((StatusCode::CREATED, "event recorded")),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct UpdateRequest {
+    entity_type: String,
+    entity_id: String,
+    field: String,
+    old_value: String,
+    new_value: String,
+    actor: String,
+    workspace_id: Option<String>,
+}
+
+async fn record_update(
+    State(journal): State<Journal>,
+    Json(req): Json<UpdateRequest>,
+) -> impl IntoResponse {
+    match journal.record_update(
+        &req.entity_type, &req.entity_id, &req.field,
+        &req.old_value, &req.new_value, &req.actor, "api",
+        &req.workspace_id.unwrap_or_default(),
+    ).await {
+        Ok(()) => Ok((StatusCode::CREATED, "update recorded")),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+// --- Flush ---
+
+async fn flush(State(journal): State<Journal>) -> impl IntoResponse {
+    match journal.flush().await {
+        Ok(n) => Ok(format!("flushed {n} events")),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+// --- Query ---
+
+async fn entity_history(
+    State(journal): State<Journal>,
+    Path(entity_id): Path<String>,
+) -> impl IntoResponse {
+    match journal.get_entity_history(&entity_id).await {
+        Ok(events) => Ok(Json(events)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct RecentQuery {
+    limit: Option<usize>,
+}
+
+async fn recent_events(
+    State(journal): State<Journal>,
+    Query(q): Query<RecentQuery>,
+) -> impl IntoResponse {
+    let limit = q.limit.unwrap_or(50);
+    match journal.get_recent(limit).await {
+        Ok(events) => Ok(Json(events)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn stats(State(journal): State<Journal>) -> impl IntoResponse {
+    Json(journal.stats().await)
+}
@@ -18,3 +18,4 @@ arrow = { workspace = true }
 bytes = { workspace = true }
 futures = { workspace = true }
 url = { workspace = true }
+chrono = { workspace = true }
@@ -0,0 +1,135 @@
+/// In-memory cache for hot datasets.
+/// Pinned datasets are loaded as Arrow RecordBatches and served from RAM.
+/// LRU eviction keeps memory bounded.
+
+use arrow::array::RecordBatch;
+use arrow::datatypes::SchemaRef;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+/// A cached dataset — schema + all batches in memory.
+#[derive(Clone)]
+pub struct CachedDataset {
+    pub name: String,
+    pub schema: SchemaRef,
+    pub batches: Vec<RecordBatch>,
+    pub row_count: usize,
+    pub size_bytes: usize, // approximate
+    pub last_accessed: std::time::Instant,
+}
+
+/// Memory cache with LRU eviction.
+#[derive(Clone)]
+pub struct MemCache {
+    datasets: Arc<RwLock<HashMap<String, CachedDataset>>>,
+    max_bytes: usize,
+}
+
+impl MemCache {
+    pub fn new(max_bytes: usize) -> Self {
+        Self {
+            datasets: Arc::new(RwLock::new(HashMap::new())),
+            max_bytes,
+        }
+    }
+
+    /// Load a dataset into the cache.
+    pub async fn put(&self, name: &str, schema: SchemaRef, batches: Vec<RecordBatch>) {
+        let row_count: usize = batches.iter().map(|b| b.num_rows()).sum();
+        let size_bytes: usize = batches.iter()
+            .map(|b| b.get_array_memory_size())
+            .sum();
+
+        // Evict if needed
+        self.evict_to_fit(size_bytes).await;
+
+        let entry = CachedDataset {
+            name: name.to_string(),
+            schema,
+            batches,
+            row_count,
+            size_bytes,
+            last_accessed: std::time::Instant::now(),
+        };
+
+        tracing::info!("cached '{}': {} rows, {:.1} MB",
+            name, row_count, size_bytes as f64 / 1024.0 / 1024.0);
+
+        let mut cache = self.datasets.write().await;
+        cache.insert(name.to_string(), entry);
+    }
+
+    /// Get a cached dataset, updating access time.
+    pub async fn get(&self, name: &str) -> Option<CachedDataset> {
+        let mut cache = self.datasets.write().await;
+        if let Some(entry) = cache.get_mut(name) {
+            entry.last_accessed = std::time::Instant::now();
+            Some(entry.clone())
+        } else {
+            None
+        }
+    }
+
+    /// Check if a dataset is cached.
+    pub async fn contains(&self, name: &str) -> bool {
+        self.datasets.read().await.contains_key(name)
+    }
+
+    /// Evict a specific dataset.
+    pub async fn evict(&self, name: &str) -> bool {
+        let mut cache = self.datasets.write().await;
+        cache.remove(name).is_some()
+    }
+
+    /// Get cache stats.
+    pub async fn stats(&self) -> CacheStats {
+        let cache = self.datasets.read().await;
+        let total_bytes: usize = cache.values().map(|d| d.size_bytes).sum();
+        let total_rows: usize = cache.values().map(|d| d.row_count).sum();
+        CacheStats {
+            datasets: cache.len(),
+            total_bytes,
+            total_rows,
+            max_bytes: self.max_bytes,
+            names: cache.keys().cloned().collect(),
+        }
+    }
+
+    /// Evict least-recently-used entries to make room for `needed_bytes`.
+    async fn evict_to_fit(&self, needed_bytes: usize) {
+        let mut cache = self.datasets.write().await;
+        let current: usize = cache.values().map(|d| d.size_bytes).sum();
+
+        if current + needed_bytes <= self.max_bytes {
+            return;
+        }
+
+        // Sort by last accessed (oldest first)
+        let mut entries: Vec<(String, std::time::Instant, usize)> = cache.iter()
+            .map(|(k, v)| (k.clone(), v.last_accessed, v.size_bytes))
+            .collect();
+        entries.sort_by_key(|(_, t, _)| *t);
+
+        let mut freed = 0usize;
+        let target = (current + needed_bytes).saturating_sub(self.max_bytes);
+
+        for (name, _, size) in entries {
+            if freed >= target {
+                break;
+            }
+            tracing::info!("evicting '{}' from cache ({:.1} MB)", name, size as f64 / 1024.0 / 1024.0);
+            cache.remove(&name);
+            freed += size;
+        }
+    }
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct CacheStats {
+    pub datasets: usize,
+    pub total_bytes: usize,
+    pub total_rows: usize,
+    pub max_bytes: usize,
+    pub names: Vec<String>,
+}
@@ -1,80 +1,131 @@
 use catalogd::registry::Registry;
 use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl};
+use datafusion::datasource::MemTable;
 use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::prelude::*;
 use object_store::ObjectStore;
 use std::sync::Arc;
 use url::Url;

+use crate::cache::MemCache;
+use crate::delta;
+
 const STORE_SCHEME: &str = "lakehouse";

-/// Query engine that runs DataFusion over catalog-registered Parquet datasets.
+/// Query engine with in-memory cache and delta merge support.
 #[derive(Clone)]
 pub struct QueryEngine {
    registry: Registry,
    store: Arc<dyn ObjectStore>,
+    cache: MemCache,
 }

 impl QueryEngine {
-    pub fn new(registry: Registry, store: Arc<dyn ObjectStore>) -> Self {
-        Self { registry, store }
+    pub fn new(registry: Registry, store: Arc<dyn ObjectStore>, cache: MemCache) -> Self {
+        Self { registry, store, cache }
    }

-    /// Execute a SQL query. Resolves all catalog datasets as tables before execution.
+    pub fn cache(&self) -> &MemCache {
+        &self.cache
+    }
+
+    pub fn store(&self) -> &Arc<dyn ObjectStore> {
+        &self.store
+    }
+
+    /// Execute a SQL query. Uses cache for hot data, falls back to Parquet.
    pub async fn query(&self, sql: &str) -> Result<Vec<arrow::array::RecordBatch>, String> {
        let ctx = self.build_context().await?;
-
        let df = ctx.sql(sql).await.map_err(|e| format!("SQL error: {e}"))?;
        let batches = df.collect().await.map_err(|e| format!("execution error: {e}"))?;
        Ok(batches)
    }

-    /// Build a SessionContext with all catalog datasets registered as tables.
-    async fn build_context(&self) -> Result<SessionContext, String> {
+    /// Pin a dataset into the memory cache.
+    pub async fn pin_dataset(&self, name: &str) -> Result<(), String> {
+        // Read from Parquet
        let ctx = SessionContext::new();
-
-        // Register the object store under a custom scheme to avoid path doubling.
-        // The store already has the root prefix (e.g. ./data), so paths are relative keys.
        let base_url = Url::parse(&format!("{STORE_SCHEME}://data/"))
            .map_err(|e| format!("invalid store url: {e}"))?;
        ctx.runtime_env().register_object_store(&base_url, self.store.clone());

-        // Register each catalog dataset as a table
+        let dataset = self.registry.get_by_name(name).await
+            .ok_or_else(|| format!("dataset not found: {name}"))?;
+
+        if dataset.objects.is_empty() {
+            return Err(format!("dataset '{name}' has no objects"));
+        }
+
+        let opts = ListingOptions::new(Arc::new(ParquetFormat::default()));
+        let table_paths: Vec<ListingTableUrl> = dataset.objects.iter()
+            .filter_map(|o| ListingTableUrl::parse(&format!("{STORE_SCHEME}://data/{}", o.key)).ok())
+            .collect();
+
+        let schema = opts.infer_schema(&ctx.state(), &table_paths[0]).await
+            .map_err(|e| format!("schema inference: {e}"))?;
+        let config = ListingTableConfig::new_with_multi_paths(table_paths)
+            .with_listing_options(opts)
+            .with_schema(schema.clone());
+        let table = ListingTable::try_new(config).map_err(|e| format!("table: {e}"))?;
+
+        ctx.register_table(name, Arc::new(table)).map_err(|e| format!("register: {e}"))?;
+        let df = ctx.sql(&format!("SELECT * FROM {name}")).await.map_err(|e| format!("read: {e}"))?;
+        let batches = df.collect().await.map_err(|e| format!("collect: {e}"))?;
+
+        self.cache.put(name, schema, batches).await;
+        Ok(())
+    }
+
+    async fn build_context(&self) -> Result<SessionContext, String> {
+        let ctx = SessionContext::new();
+
+        let base_url = Url::parse(&format!("{STORE_SCHEME}://data/"))
+            .map_err(|e| format!("invalid store url: {e}"))?;
+        ctx.runtime_env().register_object_store(&base_url, self.store.clone());
+
        let datasets = self.registry.list().await;
        for dataset in &datasets {
            if dataset.objects.is_empty() {
                continue;
            }

-            let opts = ListingOptions::new(Arc::new(ParquetFormat::default()));
+            // Check cache first
+            if let Some(cached) = self.cache.get(&dataset.name).await {
+                // Load any delta files and merge
+                let delta_batches = delta::load_deltas(&self.store, &dataset.name).await.unwrap_or_default();

-            let table_paths: Vec<ListingTableUrl> = dataset.objects.iter()
-                .filter_map(|o| {
-                    let url_str = format!("{STORE_SCHEME}://data/{}", o.key);
-                    ListingTableUrl::parse(&url_str).ok()
-                })
-                .collect();
+                let mut all_batches = cached.batches;
+                all_batches.extend(delta_batches);

+                let mem_table = MemTable::try_new(cached.schema, vec![all_batches])
+                    .map_err(|e| format!("MemTable error for {}: {e}", dataset.name))?;
+                ctx.register_table(&dataset.name, Arc::new(mem_table))
+                    .map_err(|e| format!("register cached {}: {e}", dataset.name))?;
+
+                tracing::debug!("using cached table: {}", dataset.name);
+                continue;
+            }
+
+            // Fall back to Parquet file
+            let opts = ListingOptions::new(Arc::new(ParquetFormat::default()));
+            let table_paths: Vec<ListingTableUrl> = dataset.objects.iter()
+                .filter_map(|o| ListingTableUrl::parse(&format!("{STORE_SCHEME}://data/{}", o.key)).ok())
+                .collect();
+
            if table_paths.is_empty() {
-                tracing::warn!("dataset {} has no valid paths, skipping", dataset.name);
                continue;
            }

-            // Infer schema from the first file
            let schema = opts.infer_schema(&ctx.state(), &table_paths[0]).await
                .map_err(|e| format!("schema inference failed for {}: {e}", dataset.name))?;
-
            let config = ListingTableConfig::new_with_multi_paths(table_paths)
                .with_listing_options(opts)
                .with_schema(schema);
-
            let table = ListingTable::try_new(config)
                .map_err(|e| format!("table creation failed for {}: {e}", dataset.name))?;

            ctx.register_table(&dataset.name, Arc::new(table))
                .map_err(|e| format!("table registration failed for {}: {e}", dataset.name))?;
-
-            tracing::debug!("registered table: {}", dataset.name);
        }

        Ok(ctx)
@@ -0,0 +1,131 @@
+/// Delta store for incremental updates.
+/// Instead of rewriting an entire Parquet file to change one row,
+/// we write small delta files. At query time, deltas are merged with the base.
+/// Periodic compaction merges deltas into the base file.
+
+use arrow::array::RecordBatch;
+use arrow::datatypes::SchemaRef;
+use bytes::Bytes;
+use object_store::ObjectStore;
+use std::sync::Arc;
+
+use shared::arrow_helpers::{parquet_to_record_batches, record_batch_to_parquet};
+use storaged::ops;
+
+/// Write a delta file for a dataset (new/updated rows).
+pub async fn write_delta(
+    store: &Arc<dyn ObjectStore>,
+    dataset_name: &str,
+    batch: &RecordBatch,
+) -> Result<String, String> {
+    let ts = chrono::Utc::now().timestamp_millis();
+    let key = format!("deltas/{dataset_name}/{ts}.parquet");
+    let parquet = record_batch_to_parquet(batch)?;
+    ops::put(store, &key, parquet).await?;
+    tracing::info!("wrote delta for '{}': {} rows at {}", dataset_name, batch.num_rows(), key);
+    Ok(key)
+}
+
+/// List all delta files for a dataset.
+pub async fn list_deltas(
+    store: &Arc<dyn ObjectStore>,
+    dataset_name: &str,
+) -> Result<Vec<String>, String> {
+    let prefix = format!("deltas/{dataset_name}/");
+    ops::list(store, Some(&prefix)).await
+}
+
+/// Load all delta batches for a dataset.
+pub async fn load_deltas(
+    store: &Arc<dyn ObjectStore>,
+    dataset_name: &str,
+) -> Result<Vec<RecordBatch>, String> {
+    let keys = list_deltas(store, dataset_name).await?;
+    let mut all_batches = Vec::new();
+
+    for key in &keys {
+        let data = ops::get(store, key).await?;
+        let (_, batches) = parquet_to_record_batches(&data)?;
+        all_batches.extend(batches);
+    }
+
+    if !all_batches.is_empty() {
+        let total_rows: usize = all_batches.iter().map(|b| b.num_rows()).sum();
+        tracing::debug!("loaded {} delta files ({} rows) for '{}'", keys.len(), total_rows, dataset_name);
+    }
+
+    Ok(all_batches)
+}
+
+/// Compact: merge base Parquet + all deltas into a single new base file.
+/// Optionally deduplicates by a primary key column.
+pub async fn compact(
+    store: &Arc<dyn ObjectStore>,
+    dataset_name: &str,
+    base_key: &str,
+    primary_key_col: Option<&str>,
+) -> Result<CompactResult, String> {
+    // Load base
+    let base_data = ops::get(store, base_key).await?;
+    let (schema, mut base_batches) = parquet_to_record_batches(&base_data)?;
+
+    // Load deltas
+    let delta_batches = load_deltas(store, dataset_name).await?;
+    let delta_count = delta_batches.len();
+
+    if delta_batches.is_empty() {
+        return Ok(CompactResult {
+            base_rows: base_batches.iter().map(|b| b.num_rows()).sum(),
+            delta_rows: 0,
+            final_rows: base_batches.iter().map(|b| b.num_rows()).sum(),
+            deltas_merged: 0,
+        });
+    }
+
+    base_batches.extend(delta_batches);
+
+    let base_rows: usize = base_batches.iter().map(|b| b.num_rows()).sum();
+
+    // If primary key specified, deduplicate (keep last occurrence)
+    let final_batches = if let Some(_pk) = primary_key_col {
+        // For now, just concatenate. Full dedup requires sorting by PK
+        // and keeping the last row per key — this is a simplification.
+        // TODO: implement proper merge with dedup
+        base_batches
+    } else {
+        base_batches
+    };
+
+    let final_rows: usize = final_batches.iter().map(|b| b.num_rows()).sum();
+
+    // Write merged base
+    let mut merged_parquet = Vec::new();
+    for batch in &final_batches {
+        let pq = record_batch_to_parquet(batch)?;
+        merged_parquet.extend_from_slice(&pq);
+    }
+    ops::put(store, base_key, Bytes::from(merged_parquet)).await?;
+
+    // Delete delta files
+    let delta_keys = list_deltas(store, dataset_name).await?;
+    for key in &delta_keys {
+        let _ = ops::delete(store, key).await;
+    }
+
+    tracing::info!("compacted '{}': {} deltas merged, {} → {} rows", dataset_name, delta_count, base_rows, final_rows);
+
+    Ok(CompactResult {
+        base_rows,
+        delta_rows: final_rows - base_rows + delta_count, // approximate
+        final_rows,
+        deltas_merged: delta_count,
+    })
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct CompactResult {
+    pub base_rows: usize,
+    pub delta_rows: usize,
+    pub final_rows: usize,
+    pub deltas_merged: usize,
+}
@@ -1,2 +1,7 @@
+pub mod cache;
 pub mod context;
+pub mod delta;
+pub mod paged;
 pub mod service;
+pub mod workspace;
+pub mod workspace_service;
@@ -0,0 +1,222 @@
+/// Paged query results with server-side cursor.
+/// Large results are stored server-side, client fetches pages on demand.
+/// Pattern matches the embedding supervisor: chunk → process → retry.
+
+use arrow::array::RecordBatch;
+use arrow::json::writer::{JsonArray, Writer as JsonWriter};
+use serde::Serialize;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use crate::context::QueryEngine;
+
+/// A stored query result that clients page through.
+#[derive(Clone)]
+pub struct StoredResult {
+    pub query_id: String,
+    pub sql: String,
+    pub columns: Vec<ColumnInfo>,
+    pub total_rows: usize,
+    pub batches: Vec<RecordBatch>,
+    pub created_at: std::time::Instant,
+}
+
+#[derive(Clone, Serialize)]
+pub struct ColumnInfo {
+    pub name: String,
+    pub data_type: String,
+}
+
+/// Response for initial query — metadata only, no rows.
+#[derive(Serialize)]
+pub struct QueryHandle {
+    pub query_id: String,
+    pub total_rows: usize,
+    pub columns: Vec<ColumnInfo>,
+    pub total_pages: usize,
+    pub page_size: usize,
+}
+
+/// A page of results.
+#[derive(Serialize)]
+pub struct PageResult {
+    pub query_id: String,
+    pub page: usize,
+    pub page_size: usize,
+    pub total_rows: usize,
+    pub total_pages: usize,
+    pub rows: serde_json::Value,
+    pub row_count: usize,
+}
+
+/// Server-side result store with TTL.
+#[derive(Clone)]
+pub struct ResultStore {
+    results: Arc<RwLock<HashMap<String, StoredResult>>>,
+    default_page_size: usize,
+    max_results: usize,
+}
+
+impl ResultStore {
+    pub fn new(default_page_size: usize, max_results: usize) -> Self {
+        Self {
+            results: Arc::new(RwLock::new(HashMap::new())),
+            default_page_size,
+            max_results,
+        }
+    }
+
+    /// Execute query and store results. Returns handle with metadata.
+    pub async fn execute_and_store(
+        &self,
+        engine: &QueryEngine,
+        sql: &str,
+    ) -> Result<QueryHandle, String> {
+        let batches = engine.query(sql).await?;
+
+        if batches.is_empty() {
+            let qid = format!("qr-{}", chrono::Utc::now().timestamp_millis());
+            return Ok(QueryHandle {
+                query_id: qid,
+                total_rows: 0,
+                columns: vec![],
+                total_pages: 0,
+                page_size: self.default_page_size,
+            });
+        }
+
+        let schema = batches[0].schema();
+        let columns: Vec<ColumnInfo> = schema.fields().iter().map(|f| ColumnInfo {
+            name: f.name().clone(),
+            data_type: f.data_type().to_string(),
+        }).collect();
+
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        let total_pages = (total_rows + self.default_page_size - 1) / self.default_page_size;
+
+        let qid = format!("qr-{}", chrono::Utc::now().timestamp_millis());
+
+        let result = StoredResult {
+            query_id: qid.clone(),
+            sql: sql.to_string(),
+            columns: columns.clone(),
+            total_rows,
+            batches,
+            created_at: std::time::Instant::now(),
+        };
+
+        // Store (evict old results if at capacity)
+        let mut store = self.results.write().await;
+        if store.len() >= self.max_results {
+            // Evict oldest
+            if let Some(oldest_key) = store.iter()
+                .min_by_key(|(_, v)| v.created_at)
+                .map(|(k, _)| k.clone())
+            {
+                store.remove(&oldest_key);
+            }
+        }
+        store.insert(qid.clone(), result);
+
+        Ok(QueryHandle {
+            query_id: qid,
+            total_rows,
+            columns,
+            total_pages,
+            page_size: self.default_page_size,
+        })
+    }
+
+    /// Get a page of results. Handles chunking and retry-safe.
+    pub async fn get_page(
+        &self,
+        query_id: &str,
+        page: usize,
+        page_size: Option<usize>,
+    ) -> Result<PageResult, String> {
+        let store = self.results.read().await;
+        let result = store.get(query_id)
+            .ok_or_else(|| format!("query result not found: {query_id} (may have expired)"))?;
+
+        let ps = page_size.unwrap_or(self.default_page_size);
+        let start_row = page * ps;
+
+        if start_row >= result.total_rows {
+            return Ok(PageResult {
+                query_id: query_id.to_string(),
+                page,
+                page_size: ps,
+                total_rows: result.total_rows,
+                total_pages: (result.total_rows + ps - 1) / ps,
+                rows: serde_json::Value::Array(vec![]),
+                row_count: 0,
+            });
+        }
+
+        let end_row = (start_row + ps).min(result.total_rows);
+
+        // Extract the right rows from batches
+        let page_batches = slice_batches(&result.batches, start_row, end_row);
+        let rows = batches_to_json(&page_batches)?;
+        let row_count = rows.as_array().map(|a| a.len()).unwrap_or(0);
+
+        Ok(PageResult {
+            query_id: query_id.to_string(),
+            page,
+            page_size: ps,
+            total_rows: result.total_rows,
+            total_pages: (result.total_rows + ps - 1) / ps,
+            rows,
+            row_count,
+        })
+    }
+
+    pub fn default_page_size(&self) -> usize {
+        self.default_page_size
+    }
+}
+
+/// Slice record batches to extract rows [start, end).
+fn slice_batches(batches: &[RecordBatch], start: usize, end: usize) -> Vec<RecordBatch> {
+    let mut result = Vec::new();
+    let mut current_offset = 0;
+
+    for batch in batches {
+        let batch_end = current_offset + batch.num_rows();
+
+        if batch_end <= start {
+            current_offset = batch_end;
+            continue;
+        }
+        if current_offset >= end {
+            break;
+        }
+
+        let local_start = if start > current_offset { start - current_offset } else { 0 };
+        let local_end = if end < batch_end { end - current_offset } else { batch.num_rows() };
+        let length = local_end - local_start;
+
+        if length > 0 {
+            result.push(batch.slice(local_start, length));
+        }
+
+        current_offset = batch_end;
+    }
+
+    result
+}
+
+fn batches_to_json(batches: &[RecordBatch]) -> Result<serde_json::Value, String> {
+    if batches.is_empty() {
+        return Ok(serde_json::Value::Array(vec![]));
+    }
+    let mut buf = Vec::new();
+    let mut writer = JsonWriter::<_, JsonArray>::new(&mut buf);
+    for batch in batches {
+        writer.write(batch).map_err(|e| format!("JSON write: {e}"))?;
+    }
+    writer.finish().map_err(|e| format!("JSON finish: {e}"))?;
+    drop(writer);
+    serde_json::from_slice(&buf).map_err(|e| format!("JSON parse: {e}"))
+}
@@ -2,26 +2,47 @@ use arrow::array::RecordBatch;
 use arrow::json::writer::{JsonArray, Writer as JsonWriter};
 use axum::{
    Json, Router,
-    extract::State,
+    extract::{Path, Query, State},
    http::StatusCode,
    response::IntoResponse,
    routing::{get, post},
 };
 use serde::{Deserialize, Serialize};

+use crate::cache::CacheStats;
 use crate::context::QueryEngine;
+use crate::delta;
+use crate::paged::ResultStore;
+
+#[derive(Clone)]
+pub struct QueryState {
+    pub engine: QueryEngine,
+    pub result_store: ResultStore,
+}

 pub fn router(engine: QueryEngine) -> Router {
+    let state = QueryState {
+        engine: engine.clone(),
+        result_store: ResultStore::new(100, 50), // 100 rows/page, keep 50 results
+    };
    Router::new()
        .route("/health", get(health))
        .route("/sql", post(execute_query))
-        .with_state(engine)
+        .route("/paged", post(paged_query))
+        .route("/page/{query_id}/{page}", get(get_page))
+        .route("/cache/pin", post(pin_dataset))
+        .route("/cache/evict", post(evict_dataset))
+        .route("/cache/stats", get(cache_stats))
+        .route("/compact", post(compact_dataset))
+        .with_state(state)
 }

 async fn health() -> &'static str {
    "queryd ok"
 }

+// --- SQL Query ---
+
 #[derive(Deserialize)]
 struct QueryRequest {
    sql: String,
@@ -52,12 +73,12 @@ fn batches_to_json(batches: &[RecordBatch]) -> Result<serde_json::Value, String>
 }

 async fn execute_query(
-    State(engine): State<QueryEngine>,
+    State(state): State<QueryState>,
    Json(req): Json<QueryRequest>,
 ) -> impl IntoResponse {
    tracing::info!("executing query: {}", req.sql);

-    match engine.query(&req.sql).await {
+    match state.engine.query(&req.sql).await {
        Ok(batches) => {
            if batches.is_empty() {
                return Ok(Json(QueryResponse {
@@ -87,3 +108,89 @@ async fn execute_query(
        Err(e) => Err((StatusCode::BAD_REQUEST, e)),
    }
 }
+
+// --- Paged Queries (large result sets) ---
+
+async fn paged_query(
+    State(state): State<QueryState>,
+    Json(req): Json<QueryRequest>,
+) -> impl IntoResponse {
+    tracing::info!("paged query: {}", req.sql);
+    match state.result_store.execute_and_store(&state.engine, &req.sql).await {
+        Ok(handle) => Ok(Json(handle)),
+        Err(e) => Err((StatusCode::BAD_REQUEST, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct PageQuery {
+    size: Option<usize>,
+}
+
+async fn get_page(
+    State(state): State<QueryState>,
+    Path((query_id, page)): Path<(String, usize)>,
+    Query(q): Query<PageQuery>,
+) -> impl IntoResponse {
+    match state.result_store.get_page(&query_id, page, q.size).await {
+        Ok(result) => Ok(Json(result)),
+        Err(e) => Err((StatusCode::NOT_FOUND, e)),
+    }
+}
+
+// --- Cache Management ---
+
+#[derive(Deserialize)]
+struct CacheRequest {
+    dataset: String,
+}
+
+async fn pin_dataset(
+    State(state): State<QueryState>,
+    Json(req): Json<CacheRequest>,
+) -> impl IntoResponse {
+    match state.engine.pin_dataset(&req.dataset).await {
+        Ok(()) => Ok((StatusCode::OK, format!("pinned: {}", req.dataset))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn evict_dataset(
+    State(state): State<QueryState>,
+    Json(req): Json<CacheRequest>,
+) -> impl IntoResponse {
+    if state.engine.cache().evict(&req.dataset).await {
+        (StatusCode::OK, format!("evicted: {}", req.dataset))
+    } else {
+        (StatusCode::NOT_FOUND, format!("not cached: {}", req.dataset))
+    }
+}
+
+async fn cache_stats(State(state): State<QueryState>) -> impl IntoResponse {
+    let stats = state.engine.cache().stats().await;
+    Json(stats)
+}
+
+// --- Compaction ---
+
+#[derive(Deserialize)]
+struct CompactRequest {
+    dataset: String,
+    base_key: String,
+    primary_key: Option<String>,
+}
+
+async fn compact_dataset(
+    State(state): State<QueryState>,
+    Json(req): Json<CompactRequest>,
+) -> impl IntoResponse {
+    match delta::compact(
+        state.engine.store(),
+        &req.dataset,
+        &req.base_key,
+        req.primary_key.as_deref(),
+    ).await {
+        Ok(result) => Ok(Json(result)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
@@ -0,0 +1,272 @@
+/// Agent workspaces — named overlays for contract/search-specific work.
+/// Each workspace tracks an agent's activity on a specific contract or search,
+/// with daily/weekly/monthly tiers and instant handoff capability.
+
+use arrow::array::{ArrayRef, RecordBatch, StringArray, Int64Array};
+use arrow::datatypes::{DataType, Field, Schema};
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use crate::delta;
+use object_store::ObjectStore;
+
+/// Retention tier for workspace data.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Tier {
+    Daily,   // expires end of day, active search scratch
+    Weekly,  // expires end of week, active contract work
+    Monthly, // expires end of month, contract lifecycle
+    Pinned,  // never expires, manually managed
+}
+
+/// A saved query/filter within a workspace.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SavedSearch {
+    pub name: String,
+    pub sql: String,
+    pub created_at: DateTime<Utc>,
+}
+
+/// A shortlisted candidate or record.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ShortlistEntry {
+    pub dataset: String,
+    pub record_id: String,
+    pub notes: String,
+    pub added_at: DateTime<Utc>,
+    pub added_by: String,
+}
+
+/// Activity log entry.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ActivityEntry {
+    pub action: String,      // "search", "shortlist", "call", "email", "update", "ingest"
+    pub detail: String,
+    pub timestamp: DateTime<Utc>,
+    pub agent: String,
+}
+
+/// A workspace — an agent's working context for a contract or search.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Workspace {
+    pub id: String,
+    pub name: String,            // "Apex Corp .NET Developers - Chicago"
+    pub description: String,
+    pub tier: Tier,
+    pub owner: String,           // current agent
+    pub previous_owners: Vec<HandoffRecord>,
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+
+    // Work content
+    pub saved_searches: Vec<SavedSearch>,
+    pub shortlist: Vec<ShortlistEntry>,
+    pub activity: Vec<ActivityEntry>,
+    pub ingested_datasets: Vec<String>,  // datasets this workspace created
+    pub delta_keys: Vec<String>,         // delta files specific to this workspace
+    pub tags: Vec<String>,
+}
+
+/// Record of a handoff between agents.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HandoffRecord {
+    pub from_agent: String,
+    pub to_agent: String,
+    pub reason: String,
+    pub timestamp: DateTime<Utc>,
+}
+
+/// Workspace manager — in-memory registry with persistence.
+#[derive(Clone)]
+pub struct WorkspaceManager {
+    workspaces: Arc<RwLock<HashMap<String, Workspace>>>,
+    store: Arc<dyn ObjectStore>,
+}
+
+impl WorkspaceManager {
+    pub fn new(store: Arc<dyn ObjectStore>) -> Self {
+        Self {
+            workspaces: Arc::new(RwLock::new(HashMap::new())),
+            store,
+        }
+    }
+
+    /// Rebuild from persisted workspace files on startup.
+    pub async fn rebuild(&self) -> Result<usize, String> {
+        let keys = storaged::ops::list(&self.store, Some("workspaces/")).await?;
+        let mut ws_map = self.workspaces.write().await;
+        ws_map.clear();
+
+        for key in &keys {
+            if !key.ends_with(".json") { continue; }
+            let data = storaged::ops::get(&self.store, key).await?;
+            match serde_json::from_slice::<Workspace>(&data) {
+                Ok(ws) => { ws_map.insert(ws.id.clone(), ws); }
+                Err(e) => tracing::warn!("failed to load workspace {key}: {e}"),
+            }
+        }
+
+        let count = ws_map.len();
+        if count > 0 {
+            tracing::info!("loaded {count} workspaces");
+        }
+        Ok(count)
+    }
+
+    /// Create a new workspace.
+    pub async fn create(&self, name: String, description: String, owner: String, tier: Tier) -> Result<Workspace, String> {
+        let now = Utc::now();
+        let id = format!("ws-{}", now.timestamp_millis());
+
+        let ws = Workspace {
+            id: id.clone(),
+            name,
+            description,
+            tier,
+            owner,
+            previous_owners: vec![],
+            created_at: now,
+            updated_at: now,
+            saved_searches: vec![],
+            shortlist: vec![],
+            activity: vec![],
+            ingested_datasets: vec![],
+            delta_keys: vec![],
+            tags: vec![],
+        };
+
+        self.persist(&ws).await?;
+        self.workspaces.write().await.insert(id.clone(), ws.clone());
+        tracing::info!("created workspace: {} ({})", ws.name, ws.id);
+        Ok(ws)
+    }
+
+    /// Handoff workspace to another agent. Instant — no data copy.
+    pub async fn handoff(&self, workspace_id: &str, to_agent: &str, reason: &str) -> Result<Workspace, String> {
+        let mut ws_map = self.workspaces.write().await;
+        let ws = ws_map.get_mut(workspace_id)
+            .ok_or_else(|| format!("workspace not found: {workspace_id}"))?;
+
+        let record = HandoffRecord {
+            from_agent: ws.owner.clone(),
+            to_agent: to_agent.to_string(),
+            reason: reason.to_string(),
+            timestamp: Utc::now(),
+        };
+
+        ws.previous_owners.push(record);
+        ws.owner = to_agent.to_string();
+        ws.updated_at = Utc::now();
+
+        ws.activity.push(ActivityEntry {
+            action: "handoff".to_string(),
+            detail: format!("handed off to {} — {}", to_agent, reason),
+            timestamp: Utc::now(),
+            agent: to_agent.to_string(),
+        });
+
+        let ws_clone = ws.clone();
+        drop(ws_map);
+        self.persist(&ws_clone).await?;
+
+        tracing::info!("workspace '{}' handed off to {}", ws_clone.name, to_agent);
+        Ok(ws_clone)
+    }
+
+    /// Add a saved search to a workspace.
+    pub async fn add_search(&self, workspace_id: &str, name: String, sql: String, agent: &str) -> Result<(), String> {
+        let mut ws_map = self.workspaces.write().await;
+        let ws = ws_map.get_mut(workspace_id)
+            .ok_or_else(|| format!("workspace not found: {workspace_id}"))?;
+
+        ws.saved_searches.push(SavedSearch {
+            name: name.clone(),
+            sql,
+            created_at: Utc::now(),
+        });
+        ws.activity.push(ActivityEntry {
+            action: "search".into(),
+            detail: format!("saved search: {name}"),
+            timestamp: Utc::now(),
+            agent: agent.to_string(),
+        });
+        ws.updated_at = Utc::now();
+
+        let ws_clone = ws.clone();
+        drop(ws_map);
+        self.persist(&ws_clone).await
+    }
+
+    /// Add a candidate/record to the shortlist.
+    pub async fn add_to_shortlist(&self, workspace_id: &str, dataset: String, record_id: String, notes: String, agent: &str) -> Result<(), String> {
+        let mut ws_map = self.workspaces.write().await;
+        let ws = ws_map.get_mut(workspace_id)
+            .ok_or_else(|| format!("workspace not found: {workspace_id}"))?;
+
+        ws.shortlist.push(ShortlistEntry {
+            dataset: dataset.clone(),
+            record_id: record_id.clone(),
+            notes,
+            added_at: Utc::now(),
+            added_by: agent.to_string(),
+        });
+        ws.activity.push(ActivityEntry {
+            action: "shortlist".into(),
+            detail: format!("added {record_id} from {dataset}"),
+            timestamp: Utc::now(),
+            agent: agent.to_string(),
+        });
+        ws.updated_at = Utc::now();
+
+        let ws_clone = ws.clone();
+        drop(ws_map);
+        self.persist(&ws_clone).await
+    }
+
+    /// Log an activity.
+    pub async fn log_activity(&self, workspace_id: &str, action: String, detail: String, agent: &str) -> Result<(), String> {
+        let mut ws_map = self.workspaces.write().await;
+        let ws = ws_map.get_mut(workspace_id)
+            .ok_or_else(|| format!("workspace not found: {workspace_id}"))?;
+
+        ws.activity.push(ActivityEntry {
+            action,
+            detail,
+            timestamp: Utc::now(),
+            agent: agent.to_string(),
+        });
+        ws.updated_at = Utc::now();
+
+        let ws_clone = ws.clone();
+        drop(ws_map);
+        self.persist(&ws_clone).await
+    }
+
+    /// Get a workspace.
+    pub async fn get(&self, workspace_id: &str) -> Option<Workspace> {
+        self.workspaces.read().await.get(workspace_id).cloned()
+    }
+
+    /// List all workspaces, optionally filtered by owner or tier.
+    pub async fn list(&self, owner: Option<&str>, tier: Option<&Tier>) -> Vec<Workspace> {
+        let ws_map = self.workspaces.read().await;
+        ws_map.values()
+            .filter(|ws| {
+                owner.map_or(true, |o| ws.owner == o) &&
+                tier.map_or(true, |t| ws.tier == *t)
+            })
+            .cloned()
+            .collect()
+    }
+
+    /// Persist workspace to object storage.
+    async fn persist(&self, ws: &Workspace) -> Result<(), String> {
+        let key = format!("workspaces/{}.json", ws.id);
+        let json = serde_json::to_vec_pretty(ws).map_err(|e| e.to_string())?;
+        storaged::ops::put(&self.store, &key, json.into()).await
+    }
+}
@@ -0,0 +1,136 @@
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use serde::Deserialize;
+
+use crate::workspace::{Tier, WorkspaceManager};
+
+pub fn router(manager: WorkspaceManager) -> Router {
+    Router::new()
+        .route("/", get(list_workspaces))
+        .route("/create", post(create_workspace))
+        .route("/{id}", get(get_workspace))
+        .route("/{id}/handoff", post(handoff))
+        .route("/{id}/search", post(add_search))
+        .route("/{id}/shortlist", post(add_to_shortlist))
+        .route("/{id}/activity", post(log_activity))
+        .with_state(manager)
+}
+
+#[derive(Deserialize)]
+struct ListQuery {
+    owner: Option<String>,
+    tier: Option<Tier>,
+}
+
+async fn list_workspaces(
+    State(mgr): State<WorkspaceManager>,
+    Query(q): Query<ListQuery>,
+) -> impl IntoResponse {
+    let workspaces = mgr.list(q.owner.as_deref(), q.tier.as_ref()).await;
+    Json(workspaces)
+}
+
+#[derive(Deserialize)]
+struct CreateRequest {
+    name: String,
+    description: String,
+    owner: String,
+    tier: Tier,
+}
+
+async fn create_workspace(
+    State(mgr): State<WorkspaceManager>,
+    Json(req): Json<CreateRequest>,
+) -> impl IntoResponse {
+    match mgr.create(req.name, req.description, req.owner, req.tier).await {
+        Ok(ws) => Ok((StatusCode::CREATED, Json(ws))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn get_workspace(
+    State(mgr): State<WorkspaceManager>,
+    Path(id): Path<String>,
+) -> impl IntoResponse {
+    match mgr.get(&id).await {
+        Some(ws) => Ok(Json(ws)),
+        None => Err((StatusCode::NOT_FOUND, format!("workspace not found: {id}"))),
+    }
+}
+
+#[derive(Deserialize)]
+struct HandoffRequest {
+    to_agent: String,
+    reason: String,
+}
+
+async fn handoff(
+    State(mgr): State<WorkspaceManager>,
+    Path(id): Path<String>,
+    Json(req): Json<HandoffRequest>,
+) -> impl IntoResponse {
+    match mgr.handoff(&id, &req.to_agent, &req.reason).await {
+        Ok(ws) => Ok(Json(ws)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct SearchRequest {
+    name: String,
+    sql: String,
+    agent: String,
+}
+
+async fn add_search(
+    State(mgr): State<WorkspaceManager>,
+    Path(id): Path<String>,
+    Json(req): Json<SearchRequest>,
+) -> impl IntoResponse {
+    match mgr.add_search(&id, req.name, req.sql, &req.agent).await {
+        Ok(()) => Ok((StatusCode::OK, "search saved")),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct ShortlistRequest {
+    dataset: String,
+    record_id: String,
+    notes: String,
+    agent: String,
+}
+
+async fn add_to_shortlist(
+    State(mgr): State<WorkspaceManager>,
+    Path(id): Path<String>,
+    Json(req): Json<ShortlistRequest>,
+) -> impl IntoResponse {
+    match mgr.add_to_shortlist(&id, req.dataset, req.record_id, req.notes, &req.agent).await {
+        Ok(()) => Ok((StatusCode::OK, "added to shortlist")),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct ActivityRequest {
+    action: String,
+    detail: String,
+    agent: String,
+}
+
+async fn log_activity(
+    State(mgr): State<WorkspaceManager>,
+    Path(id): Path<String>,
+    Json(req): Json<ActivityRequest>,
+) -> impl IntoResponse {
+    match mgr.log_activity(&id, req.action, req.detail, &req.agent).await {
+        Ok(()) => Ok((StatusCode::OK, "activity logged")),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
@@ -2,3 +2,4 @@ pub mod types;
 pub mod errors;
 pub mod arrow_helpers;
 pub mod config;
+pub mod pii;
@@ -0,0 +1,116 @@
+/// Auto-detect PII columns by name patterns.
+/// Conservative: flags likely PII, doesn't miss obvious cases.
+
+use crate::types::Sensitivity;
+
+/// Check if a column name suggests PII content.
+pub fn detect_sensitivity(column_name: &str) -> Option<Sensitivity> {
+    let lower = column_name.to_lowercase();
+
+    // Direct PII identifiers
+    if matches!(lower.as_str(),
+        "ssn" | "social_security" | "social_security_number" |
+        "sin" | "national_id" | "passport" | "passport_number" |
+        "drivers_license" | "driver_license" | "dl_number"
+    ) {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Names
+    if lower.contains("first_name") || lower.contains("last_name") ||
+       lower.contains("full_name") || lower.contains("middle_name") ||
+       lower == "name" || lower == "fname" || lower == "lname" {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Contact info
+    if lower.contains("email") || lower.contains("e_mail") ||
+       lower.contains("phone") || lower.contains("mobile") || lower.contains("cell") ||
+       lower.contains("fax") || lower == "tel" || lower == "telephone" {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Address
+    if lower.contains("address") || lower.contains("street") ||
+       (lower.contains("zip") && !lower.contains("unzip")) ||
+       lower == "postal_code" || lower == "postcode" {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Financial
+    if lower.contains("salary") || lower.contains("wage") ||
+       lower.contains("pay_rate") || lower.contains("bill_rate") ||
+       lower.contains("compensation") || lower.contains("revenue") ||
+       lower.contains("bank_account") || lower.contains("routing_number") ||
+       lower.contains("credit_card") {
+        return Some(Sensitivity::Financial);
+    }
+
+    // Health
+    if lower.contains("diagnosis") || lower.contains("medication") ||
+       lower.contains("medical") || lower.contains("health") ||
+       lower.contains("patient_id") || lower.contains("mrn") {
+        return Some(Sensitivity::Phi);
+    }
+
+    // Date of birth
+    if lower == "dob" || lower == "date_of_birth" || lower == "birthdate" || lower == "birth_date" {
+        return Some(Sensitivity::Pii);
+    }
+
+    None
+}
+
+/// Classify all columns and return the highest sensitivity found.
+pub fn detect_dataset_sensitivity(column_names: &[&str]) -> Option<Sensitivity> {
+    let mut highest: Option<Sensitivity> = None;
+
+    for name in column_names {
+        if let Some(sens) = detect_sensitivity(name) {
+            highest = Some(match (&highest, &sens) {
+                (None, s) => s.clone(),
+                (Some(Sensitivity::Public), s) => s.clone(),
+                (Some(Sensitivity::Internal), s) if !matches!(s, Sensitivity::Public | Sensitivity::Internal) => s.clone(),
+                (Some(Sensitivity::Financial), Sensitivity::Pii | Sensitivity::Phi) => sens.clone(),
+                (Some(Sensitivity::Pii), Sensitivity::Phi) => Sensitivity::Phi,
+                (existing, _) => existing.clone().unwrap(),
+            });
+        }
+    }
+
+    highest
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn detects_email_as_pii() {
+        assert_eq!(detect_sensitivity("email"), Some(Sensitivity::Pii));
+        assert_eq!(detect_sensitivity("contact_email"), Some(Sensitivity::Pii));
+    }
+
+    #[test]
+    fn detects_salary_as_financial() {
+        assert_eq!(detect_sensitivity("salary"), Some(Sensitivity::Financial));
+        assert_eq!(detect_sensitivity("bill_rate"), Some(Sensitivity::Financial));
+    }
+
+    #[test]
+    fn detects_ssn() {
+        assert_eq!(detect_sensitivity("ssn"), Some(Sensitivity::Pii));
+    }
+
+    #[test]
+    fn non_sensitive_returns_none() {
+        assert_eq!(detect_sensitivity("status"), None);
+        assert_eq!(detect_sensitivity("created_at"), None);
+    }
+
+    #[test]
+    fn dataset_sensitivity_picks_highest() {
+        let cols = vec!["id", "name", "email", "status"];
+        assert_eq!(detect_dataset_sensitivity(&cols), Some(Sensitivity::Pii));
+    }
+}
@@ -30,7 +30,50 @@ pub struct ObjectRef {
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct SchemaFingerprint(pub String);

-/// Dataset manifest — the catalog's view of a dataset.
+/// Sensitivity classification for a field or dataset.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Sensitivity {
+    Public,      // anyone can see
+    Internal,    // employees only
+    Pii,         // personally identifiable (name, email, phone, SSN)
+    Phi,         // protected health info (HIPAA)
+    Financial,   // compensation, billing, revenue
+}
+
+/// Column-level metadata.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ColumnMeta {
+    pub name: String,
+    pub data_type: String,
+    #[serde(default)]
+    pub sensitivity: Option<Sensitivity>,
+    #[serde(default)]
+    pub description: String,
+    #[serde(default)]
+    pub is_pii: bool,
+}
+
+/// Data lineage — where this dataset came from.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Lineage {
+    pub source_system: String,    // "csv_upload", "mysql_import", "api_ingest"
+    pub source_file: String,      // original filename or URI
+    pub ingest_job: String,       // job ID that created this dataset
+    pub ingest_timestamp: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub parent_datasets: Vec<String>, // derived from these datasets
+}
+
+/// Freshness contract — how often this data should be updated.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FreshnessContract {
+    pub expected_frequency: String, // "daily", "weekly", "hourly", "manual"
+    pub stale_after_hours: u32,     // alert if not updated within this many hours
+}
+
+/// Dataset manifest — the catalog's complete view of a dataset.
+/// All new fields use #[serde(default)] for backward compatibility with existing manifests.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DatasetManifest {
    pub id: DatasetId,
@@ -39,4 +82,38 @@ pub struct DatasetManifest {
    pub objects: Vec<ObjectRef>,
    pub created_at: chrono::DateTime<chrono::Utc>,
    pub updated_at: chrono::DateTime<chrono::Utc>,
+
+    // --- Rich metadata (Phase 10) ---
+
+    /// Human-readable description (auto-generated on ingest or manual)
+    #[serde(default)]
+    pub description: String,
+
+    /// Who owns this dataset
+    #[serde(default)]
+    pub owner: String,
+
+    /// Overall sensitivity classification
+    #[serde(default)]
+    pub sensitivity: Option<Sensitivity>,
+
+    /// Column-level metadata including sensitivity and descriptions
+    #[serde(default)]
+    pub columns: Vec<ColumnMeta>,
+
+    /// Where this data came from
+    #[serde(default)]
+    pub lineage: Option<Lineage>,
+
+    /// How fresh the data should be
+    #[serde(default)]
+    pub freshness: Option<FreshnessContract>,
+
+    /// Free-form tags for discovery
+    #[serde(default)]
+    pub tags: Vec<String>,
+
+    /// Row count (updated on ingest/compact)
+    #[serde(default)]
+    pub row_count: Option<u64>,
 }
@@ -162,3 +162,46 @@ th {
 }
 td { padding: 8px 14px; border-bottom: 1px solid var(--border); }
 tr:hover td { background: var(--accent-glow); }
+
+/* Dashboard */
+.dashboard-hero { margin-bottom: 28px; }
+.dashboard-hero h2 { font-size: 32px; font-weight: 700; }
+
+.stat-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); gap: 12px; margin-bottom: 32px; }
+.stat-card {
+    background: var(--surface); border: 1px solid var(--border); border-radius: 10px;
+    padding: 20px 16px; text-align: center;
+}
+.stat-card.accent { border-color: var(--accent); background: var(--accent-glow); }
+.stat-value { font-size: 28px; font-weight: 700; color: var(--text); }
+.stat-card.accent .stat-value { color: var(--accent); }
+.stat-label { font-size: 11px; color: var(--text-dim); margin-top: 4px; text-transform: uppercase; letter-spacing: 0.05em; }
+
+.architecture-section, .phases-section { margin-bottom: 28px; }
+.architecture-section h3, .phases-section h3 { font-size: 16px; font-weight: 600; color: var(--text-dim); margin-bottom: 12px; text-transform: uppercase; letter-spacing: 0.08em; }
+
+.arch-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 10px; }
+.arch-card {
+    background: var(--surface); border: 1px solid var(--border); border-radius: 8px;
+    padding: 14px; border-left: 3px solid var(--accent);
+}
+.arch-title { font-weight: 600; font-size: 13px; margin-bottom: 4px; color: var(--accent); }
+.arch-items { font-size: 11px; color: var(--text-dim); line-height: 1.6; }
+
+.phase-list { display: flex; flex-direction: column; gap: 2px; }
+.phase-item {
+    display: grid; grid-template-columns: 40px 180px 1fr; gap: 12px; align-items: center;
+    padding: 8px 12px; border-radius: 6px; font-size: 12px;
+    background: var(--surface); border-left: 3px solid var(--success);
+}
+.phase-num { font-weight: 700; color: var(--accent); text-align: center; }
+.phase-name { font-weight: 600; }
+.phase-detail { color: var(--text-dim); }
+
+/* Ingest */
+.table-list { margin-top: 12px; }
+.table-item {
+    display: flex; justify-content: space-between; align-items: center;
+    padding: 8px 12px; border-bottom: 1px solid var(--border); font-size: 13px;
+}
+.table-item:hover { background: var(--accent-glow); }
@@ -0,0 +1,23 @@
+{
+  "id": "6d0002ef-28de-4b92-abed-0bfb7ab5fb6f",
+  "name": "candidates",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/candidates.parquet",
+      "size_bytes": 10592165,
+      "created_at": "2026-03-28T01:54:06.898505222Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:06.898506121Z",
+  "updated_at": "2026-03-28T01:54:06.898506121Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "73a4ef71-5b2a-4124-a2c2-fcefdde99624",
+  "name": "job_orders",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/job_orders.parquet",
+      "size_bytes": 905534,
+      "created_at": "2026-03-28T01:54:07.007332060Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:07.007332775Z",
+  "updated_at": "2026-03-28T01:54:07.007332775Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "b56694bb-f4f7-447d-b4d6-ac89103a6a3a",
+  "name": "call_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/call_log.parquet",
+      "size_bytes": 35951077,
+      "created_at": "2026-03-28T01:54:15.457573021Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:15.457573701Z",
+  "updated_at": "2026-03-28T01:54:15.457573701Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "daa5a11b-6898-40ba-a811-3bd4b6f5b750",
+  "name": "email_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/email_log.parquet",
+      "size_bytes": 16768671,
+      "created_at": "2026-03-28T01:54:17.356056562Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:17.356057529Z",
+  "updated_at": "2026-03-28T01:54:17.356057529Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "dd396ee4-3a1a-4939-af60-84d409eebed4",
+  "name": "clients",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/clients.parquet",
+      "size_bytes": 21971,
+      "created_at": "2026-03-28T01:54:06.904379300Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:06.904379793Z",
+  "updated_at": "2026-03-28T01:54:06.904379793Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "f85285e3-111a-4a55-aa58-05fd740ed862",
+  "name": "timesheets",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/timesheets.parquet",
+      "size_bytes": 17539932,
+      "created_at": "2026-03-28T01:54:11.947773535Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:11.947778065Z",
+  "updated_at": "2026-03-28T01:54:11.947778065Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "ffa72582-dc11-4454-a46e-35839a7d04bb",
+  "name": "placements",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/placements.parquet",
+      "size_bytes": 1213820,
+      "created_at": "2026-03-28T01:54:07.109218264Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:54:07.109219070Z",
+  "updated_at": "2026-03-28T01:54:07.109219070Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -109,20 +109,43 @@ async fn fetch_health(path: &str) -> Result<String, String> {
    resp.text().await.map_err(|e| e.to_string())
 }

-/// Get schema context for all datasets (used for AI SQL generation)
+/// Cached schema context — built once, reused across questions.
+static SCHEMA_CACHE: std::sync::OnceLock<std::sync::Mutex<Option<String>>> = std::sync::OnceLock::new();
+
+async fn get_schema_context_cached(datasets: &[Dataset]) -> String {
+    // Check cache first
+    {
+        let cache = SCHEMA_CACHE.get_or_init(|| std::sync::Mutex::new(None));
+        if let Ok(guard) = cache.lock() {
+            if let Some(ref cached) = *guard {
+                return cached.clone();
+            }
+        }
+    }
+    // Build and cache
+    let ctx = get_schema_context(datasets).await;
+    if let Some(cache) = SCHEMA_CACHE.get() {
+        if let Ok(mut guard) = cache.lock() {
+            *guard = Some(ctx.clone());
+        }
+    }
+    ctx
+}
+
+/// Get schema context for datasets (used for AI SQL generation).
 async fn get_schema_context(datasets: &[Dataset]) -> String {
-    let mut ctx = String::from("Available tables:\n\n");
-    for ds in datasets {
+    let core_tables = ["candidates", "clients", "job_orders", "placements", "timesheets", "call_log", "email_log"];
+    let mut ctx = String::from("DATABASE SCHEMA:\n\n");
+    for ds in datasets.iter().filter(|d| core_tables.contains(&d.name.as_str())) {
        let desc = run_sql(&format!("DESCRIBE {}", ds.name)).await;
        match desc {
            Ok(resp) => {
-                ctx.push_str(&format!("TABLE: {}\n", ds.name));
+                ctx.push_str(&format!("TABLE: {}\n  Columns:\n", ds.name));
                if let Some(rows) = resp.rows.as_array() {
                    for row in rows {
                        let col = row.get("column_name").and_then(|v| v.as_str()).unwrap_or("?");
                        let dt = row.get("data_type").and_then(|v| v.as_str()).unwrap_or("?");
-                        let nullable = row.get("is_nullable").and_then(|v| v.as_str()).unwrap_or("?");
-                        ctx.push_str(&format!("  - {} ({}, nullable={})\n", col, dt, nullable));
+                        ctx.push_str(&format!("    {}.{} ({})\n", ds.name, col, dt));
                    }
                }
                ctx.push('\n');
@@ -132,15 +155,29 @@ async fn get_schema_context(datasets: &[Dataset]) -> String {
            }
        }
    }
+    // Add relationship hints so the model knows how to JOIN
+    ctx.push_str("RELATIONSHIPS:\n");
+    ctx.push_str("  candidates.candidate_id = placements.candidate_id\n");
+    ctx.push_str("  candidates.candidate_id = timesheets.candidate_id\n");
+    ctx.push_str("  candidates.candidate_id = call_log.candidate_id\n");
+    ctx.push_str("  candidates.candidate_id = email_log.candidate_id\n");
+    ctx.push_str("  clients.client_id = job_orders.client_id\n");
+    ctx.push_str("  clients.client_id = placements.client_id\n");
+    ctx.push_str("  clients.client_id = timesheets.client_id\n");
+    ctx.push_str("  job_orders.job_order_id = placements.job_order_id\n");
+    ctx.push_str("  placements.placement_id = timesheets.placement_id\n");
+    ctx.push_str("\nNOTE: 'vertical' is only in candidates, clients, and job_orders. To get vertical for timesheets or placements, JOIN to those tables.\n");
    ctx
 }

 // --- Tabs ---
 #[derive(Clone, PartialEq)]
 enum Tab {
+    Dashboard,
    Ask,
    Explore,
    Sql,
+    Ingest,
    Status,
 }

@@ -177,6 +214,11 @@ fn App() -> Element {
            div { class: "header",
                h1 { "LAKEHOUSE" }
                div { class: "tabs",
+                    button {
+                        class: if *active_tab.read() == Tab::Dashboard { "tab active" } else { "tab" },
+                        onclick: move |_| active_tab.set(Tab::Dashboard),
+                        "Dashboard"
+                    }
                    button {
                        class: if *active_tab.read() == Tab::Ask { "tab active" } else { "tab" },
                        onclick: move |_| active_tab.set(Tab::Ask),
@@ -192,6 +234,11 @@ fn App() -> Element {
                        onclick: move |_| active_tab.set(Tab::Sql),
                        "SQL"
                    }
+                    button {
+                        class: if *active_tab.read() == Tab::Ingest { "tab active" } else { "tab" },
+                        onclick: move |_| active_tab.set(Tab::Ingest),
+                        "Ingest"
+                    }
                    button {
                        class: if *active_tab.read() == Tab::Status { "tab active" } else { "tab" },
                        onclick: move |_| active_tab.set(Tab::Status),
@@ -208,9 +255,11 @@ fn App() -> Element {

            div { class: "content-full",
                match *active_tab.read() {
+                    Tab::Dashboard => rsx! { DashboardPanel {} },
                    Tab::Ask => rsx! { AskPanel { datasets: datasets.read().clone() } },
                    Tab::Explore => rsx! { ExplorePanel { datasets: datasets.read().clone() } },
                    Tab::Sql => rsx! { SqlPanel {} },
+                    Tab::Ingest => rsx! { IngestPanel {} },
                    Tab::Status => rsx! { StatusPanel {} },
                }
            }
@@ -241,26 +290,55 @@ fn AskPanel(datasets: Vec<Dataset>) -> Element {

            // Step 1: Get schema context
            step.set("reading schemas...".into());
-            let schema_ctx = get_schema_context(&ds).await;
+            let schema_ctx = get_schema_context_cached(&ds).await;

            // Step 2: Generate SQL
            step.set("writing SQL...".into());
            let prompt = format!(
-                "You are a SQL assistant for a data lakehouse using Apache DataFusion (PostgreSQL-compatible SQL).\n\n\
+                "You are a SQL assistant. You write Apache DataFusion SQL (PostgreSQL-compatible).\n\n\
+                CRITICAL: You MUST only use column names that appear in the schema below. Do NOT invent or guess column names.\n\n\
                {schema_ctx}\n\
                User question: {q}\n\n\
-                Write a SQL query that answers this question. Output ONLY the SQL query, nothing else. No markdown, no explanation, no backticks."
+                Rules:\n\
+                - ONLY use table and column names listed above\n\
+                - If unsure about a column name, pick the closest match from the schema\n\
+                - Output ONLY the SQL query. No markdown, no explanation, no backticks."
            );

            match ai_generate(&prompt, 512).await {
                Ok(resp) => {
-                    let sql = resp.text.trim().to_string();
+                    let sql = clean_sql(&resp.text);
                    generated_sql.set(Some(sql.clone()));

                    // Step 3: Execute
                    step.set("running query...".into());
                    let query_result = run_sql(&sql).await;
-                    result.set(Some(query_result));
+
+                    // Step 3b: If schema error, retry with the error as feedback
+                    if let Err(ref err) = query_result {
+                        if err.contains("error") {
+                            step.set("fixing SQL...".into());
+                            let retry_prompt = format!(
+                                "The SQL you wrote had an error:\n{err}\n\n\
+                                {schema_ctx}\n\
+                                Original question: {q}\n\n\
+                                Write a CORRECTED SQL query using ONLY the columns listed in the schema. Output ONLY SQL."
+                            );
+                            if let Ok(retry_resp) = ai_generate(&retry_prompt, 512).await {
+                                let retry_sql = clean_sql(&retry_resp.text);
+                                generated_sql.set(Some(retry_sql.clone()));
+                                step.set("running corrected query...".into());
+                                let retry_result = run_sql(&retry_sql).await;
+                                result.set(Some(retry_result));
+                            } else {
+                                result.set(Some(query_result));
+                            }
+                        } else {
+                            result.set(Some(query_result));
+                        }
+                    } else {
+                        result.set(Some(query_result));
+                    }
                }
                Err(e) => {
                    result.set(Some(Err(format!("AI error: {e}"))));
@@ -295,21 +373,44 @@ fn AskPanel(datasets: Vec<Dataset>) -> Element {
                                generated_sql.set(None);
                                result.set(None);
                                step.set("reading schemas...".into());
-                                let schema_ctx = get_schema_context(&ds).await;
+                                let schema_ctx = get_schema_context_cached(&ds).await;
                                step.set("writing SQL...".into());
                                let prompt = format!(
-                                    "You are a SQL assistant for a data lakehouse using Apache DataFusion (PostgreSQL-compatible SQL).\n\n\
+                                    "You are a SQL assistant. You write Apache DataFusion SQL (PostgreSQL-compatible).\n\n\
+                                    CRITICAL: You MUST only use column names that appear in the schema below. Do NOT invent or guess column names.\n\n\
                                    {schema_ctx}\n\
                                    User question: {q}\n\n\
-                                    Write a SQL query that answers this question. Output ONLY the SQL query, nothing else. No markdown, no explanation, no backticks."
+                                    Rules:\n\
+                                    - ONLY use table and column names listed above\n\
+                                    - If unsure about a column name, pick the closest match from the schema\n\
+                                    - Output ONLY the SQL query. No markdown, no explanation, no backticks."
                                );
                                match ai_generate(&prompt, 512).await {
                                    Ok(resp) => {
-                                        let sql = resp.text.trim().to_string();
+                                        let sql = clean_sql(&resp.text);
                                        generated_sql.set(Some(sql.clone()));
                                        step.set("running query...".into());
                                        let query_result = run_sql(&sql).await;
-                                        result.set(Some(query_result));
+                                        if let Err(ref err) = query_result {
+                                            if err.contains("error") {
+                                                step.set("fixing SQL...".into());
+                                                let retry_prompt = format!(
+                                                    "The SQL you wrote had an error:\n{err}\n\n{schema_ctx}\n\nOriginal question: {q}\n\nWrite a CORRECTED SQL query using ONLY the columns listed. Output ONLY SQL."
+                                                );
+                                                if let Ok(rr) = ai_generate(&retry_prompt, 512).await {
+                                                    let rsql = clean_sql(&rr.text);
+                                                    generated_sql.set(Some(rsql.clone()));
+                                                    step.set("running corrected query...".into());
+                                                    result.set(Some(run_sql(&rsql).await));
+                                                } else {
+                                                    result.set(Some(query_result));
+                                                }
+                                            } else {
+                                                result.set(Some(query_result));
+                                            }
+                                        } else {
+                                            result.set(Some(query_result));
+                                        }
                                    }
                                    Err(e) => {
                                        result.set(Some(Err(format!("AI error: {e}"))));
@@ -532,6 +633,303 @@ fn SqlPanel() -> Element {
    }
 }

+// === DASHBOARD — System overview ===
+
+#[component]
+fn DashboardPanel() -> Element {
+    let mut stats = use_signal(|| None::<serde_json::Value>);
+    let mut loading = use_signal(|| false);
+
+    let do_load = move || {
+        spawn(async move {
+            loading.set(true);
+            let client = reqwest::Client::new();
+
+            let datasets = fetch_datasets().await.ok().map(|d| d.len()).unwrap_or(0);
+            let row_count = run_sql("SELECT (SELECT COUNT(*) FROM candidates) + (SELECT COUNT(*) FROM clients) + (SELECT COUNT(*) FROM job_orders) + (SELECT COUNT(*) FROM placements) + (SELECT COUNT(*) FROM timesheets) + (SELECT COUNT(*) FROM call_log) + (SELECT COUNT(*) FROM email_log) as total").await.ok().and_then(|r| r.rows.as_array()?.first()?.get("total")?.as_i64()).unwrap_or(0);
+
+            let cache: Option<serde_json::Value> = match client.get(&format!("{}/query/cache/stats", api_base())).send().await {
+                Ok(r) => r.json().await.ok(),
+                Err(_) => None,
+            };
+
+            let indexes: Vec<serde_json::Value> = match client.get(&format!("{}/vectors/indexes", api_base())).send().await {
+                Ok(r) => r.json().await.unwrap_or_default(),
+                Err(_) => vec![],
+            };
+
+            let tools: Vec<serde_json::Value> = match client.get(&format!("{}/tools", api_base())).send().await {
+                Ok(r) => r.json().await.unwrap_or_default(),
+                Err(_) => vec![],
+            };
+
+            let jobs: Vec<serde_json::Value> = match client.get(&format!("{}/vectors/jobs", api_base())).send().await {
+                Ok(r) => r.json().await.unwrap_or_default(),
+                Err(_) => vec![],
+            };
+
+            let hnsw: Vec<String> = match client.get(&format!("{}/vectors/hnsw/list", api_base())).send().await {
+                Ok(r) => r.json().await.unwrap_or_default(),
+                Err(_) => vec![],
+            };
+
+            let journal: Option<serde_json::Value> = match client.get(&format!("{}/journal/stats", api_base())).send().await {
+                Ok(r) => r.json().await.ok(),
+                Err(_) => None,
+            };
+
+            stats.set(Some(serde_json::json!({
+                "datasets": datasets,
+                "total_rows": row_count,
+                "cache": cache,
+                "vector_indexes": indexes.len(),
+                "vector_total_chunks": indexes.iter().filter_map(|i| i.get("chunk_count")?.as_i64()).sum::<i64>(),
+                "hnsw_loaded": hnsw.len(),
+                "tools": tools.len(),
+                "jobs_total": jobs.len(),
+                "jobs_running": jobs.iter().filter(|j| j.get("status").and_then(|s| s.as_str()) == Some("running")).count(),
+                "journal": journal,
+            })));
+            loading.set(false);
+        });
+    };
+
+    // Auto-load on mount
+    // Load on first render (user clicks Dashboard tab)
+    let mut loaded = use_signal(|| false);
+    use_effect(move || {
+        if !*loaded.read() {
+            loaded.set(true);
+            do_load();
+        }
+    });
+
+    rsx! {
+        div { class: "panel",
+            div { class: "dashboard-hero",
+                h2 { "Lakehouse" }
+                p { class: "subtitle", "Rust-first data platform — SQL + AI + RAG over object storage" }
+            }
+
+            if let Some(s) = stats.read().as_ref() {
+                div { class: "stat-grid",
+                    div { class: "stat-card",
+                        div { class: "stat-value", "{s[\"datasets\"]}" }
+                        div { class: "stat-label", "Datasets" }
+                    }
+                    div { class: "stat-card",
+                        div { class: "stat-value", {format!("{:.1}M", s["total_rows"].as_i64().unwrap_or(0) as f64 / 1_000_000.0)} }
+                        div { class: "stat-label", "Total Rows" }
+                    }
+                    div { class: "stat-card",
+                        div { class: "stat-value", {format!("{}K", s["vector_total_chunks"].as_i64().unwrap_or(0) / 1000)} }
+                        div { class: "stat-label", "Embeddings" }
+                    }
+                    div { class: "stat-card accent",
+                        div { class: "stat-value", "{s[\"hnsw_loaded\"]}" }
+                        div { class: "stat-label", "HNSW Indexes (27ms)" }
+                    }
+                    div { class: "stat-card",
+                        div { class: "stat-value", "{s[\"tools\"]}" }
+                        div { class: "stat-label", "Governed Tools" }
+                    }
+                    div { class: "stat-card",
+                        div { class: "stat-value", {
+                            s.get("cache")
+                                .and_then(|c| c.get("datasets"))
+                                .and_then(|d| d.as_i64())
+                                .map(|n| format!("{n}"))
+                                .unwrap_or("0".into())
+                        } }
+                        div { class: "stat-label", "Cached Datasets" }
+                    }
+                }
+
+                div { class: "architecture-section",
+                    h3 { "Architecture" }
+                    div { class: "arch-grid",
+                        div { class: "arch-card",
+                            div { class: "arch-title", "Ingest" }
+                            div { class: "arch-items", "CSV, JSON, PDF, Text, PostgreSQL, File Watcher" }
+                        }
+                        div { class: "arch-card",
+                            div { class: "arch-title", "Storage" }
+                            div { class: "arch-items", "Parquet on Object Storage, Delta Writes, Compaction" }
+                        }
+                        div { class: "arch-card",
+                            div { class: "arch-title", "Query" }
+                            div { class: "arch-items", "DataFusion SQL, MemCache (9.8x), Hot/Cold" }
+                        }
+                        div { class: "arch-card",
+                            div { class: "arch-title", "AI" }
+                            div { class: "arch-items", "Ollama (local), Embed, Generate, RAG, HNSW" }
+                        }
+                        div { class: "arch-card",
+                            div { class: "arch-title", "Governance" }
+                            div { class: "arch-items", "Event Journal, PII Detection, Tool Registry, Access Control" }
+                        }
+                        div { class: "arch-card",
+                            div { class: "arch-title", "Agents" }
+                            div { class: "arch-items", "Workspaces, Handoff, Shortlists, Activity Logs" }
+                        }
+                    }
+                }
+
+                div { class: "phases-section",
+                    h3 { "Build Progression" }
+                    div { class: "phase-list",
+                        {rsx! {
+                            PhaseItem { num: "0-5", name: "Foundation", detail: "Storage, Catalog, DataFusion, AI, UI, gRPC" }
+                            PhaseItem { num: "6", name: "Ingest Pipeline", detail: "CSV/JSON/PDF/Text auto-schema" }
+                            PhaseItem { num: "7", name: "Vector + RAG", detail: "Embed, Search, LLM Answers" }
+                            PhaseItem { num: "8", name: "Hot Cache", detail: "9.8x speedup, Delta Writes" }
+                            PhaseItem { num: "8.5", name: "Agent Workspaces", detail: "Per-contract, Instant Handoff" }
+                            PhaseItem { num: "9", name: "Event Journal", detail: "Append-only Mutation History" }
+                            PhaseItem { num: "10", name: "Rich Catalog", detail: "PII Detection, Lineage" }
+                            PhaseItem { num: "11", name: "Embedding Versioning", detail: "Model-proof Vectors" }
+                            PhaseItem { num: "12", name: "Tool Registry", detail: "6 Governed Actions + Audit" }
+                            PhaseItem { num: "13", name: "Access Control", detail: "Role-based, Field-level" }
+                            PhaseItem { num: "14", name: "Schema Evolution", detail: "Diff Detection, AI Migration" }
+                            PhaseItem { num: "15", name: "HNSW Index", detail: "100K Search in 27ms" }
+                            PhaseItem { num: "16", name: "File Watcher", detail: "Auto-ingest from Inbox" }
+                            PhaseItem { num: "17", name: "DB Connector", detail: "PostgreSQL Import" }
+                        }}
+                    }
+                }
+            } else if *loading.read() {
+                div { class: "loading", "loading dashboard..." }
+            }
+
+            button { class: "btn", onclick: move |_| do_load(), "Refresh" }
+        }
+    }
+}
+
+#[component]
+fn PhaseItem(num: String, name: String, detail: String) -> Element {
+    rsx! {
+        div { class: "phase-item",
+            span { class: "phase-num", "{num}" }
+            span { class: "phase-name", "{name}" }
+            span { class: "phase-detail", "{detail}" }
+        }
+    }
+}
+
+// === INGEST — Data on-ramp ===
+
+#[component]
+fn IngestPanel() -> Element {
+    let mut pg_host = use_signal(|| "127.0.0.1".to_string());
+    let mut pg_db = use_signal(|| "knowledge_base".to_string());
+    let mut pg_tables = use_signal(|| None::<Vec<String>>);
+    let mut pg_result = use_signal(|| None::<Result<serde_json::Value, String>>);
+    let mut pg_loading = use_signal(|| false);
+
+    let list_tables = move |_| {
+        let host = pg_host.read().clone();
+        let db = pg_db.read().clone();
+        spawn(async move {
+            pg_loading.set(true);
+            let client = reqwest::Client::new();
+            let resp = client.post(&format!("{}/ingest/postgres/tables", api_base()))
+                .json(&serde_json::json!({"host": host, "port": 5432, "database": db, "user": "postgres", "password": ""}))
+                .send().await;
+            match resp {
+                Ok(r) => {
+                    if let Ok(tables) = r.json::<Vec<String>>().await {
+                        pg_tables.set(Some(tables));
+                    }
+                }
+                Err(e) => pg_tables.set(None),
+            }
+            pg_loading.set(false);
+        });
+    };
+
+    let mut import_table = move |table: String| {
+        let host = pg_host.read().clone();
+        let db = pg_db.read().clone();
+        spawn(async move {
+            pg_result.set(None);
+            let client = reqwest::Client::new();
+            let resp = client.post(&format!("{}/ingest/postgres/import", api_base()))
+                .json(&serde_json::json!({"host": host, "port": 5432, "database": db, "user": "postgres", "password": "", "table": table}))
+                .send().await;
+            match resp {
+                Ok(r) => {
+                    match r.json::<serde_json::Value>().await {
+                        Ok(v) => pg_result.set(Some(Ok(v))),
+                        Err(e) => pg_result.set(Some(Err(e.to_string()))),
+                    }
+                }
+                Err(e) => pg_result.set(Some(Err(e.to_string()))),
+            }
+        });
+    };
+
+    rsx! {
+        div { class: "panel",
+            h2 { "Data Ingest" }
+            p { class: "subtitle", "Bring data in from files, databases, or the auto-watch inbox" }
+
+            div { class: "panel-section",
+                h3 { "File Upload" }
+                p { class: "hint", "POST a file to /ingest/file — or drop it in ./inbox/ for auto-ingest" }
+                div { class: "arch-card",
+                    div { class: "arch-title", "Supported Formats" }
+                    div { class: "arch-items", "CSV (auto-schema) | JSON (nested flattening) | PDF (text extraction) | Text/SMS" }
+                }
+                div { class: "arch-card",
+                    div { class: "arch-title", "Auto-Watch Inbox" }
+                    div { class: "arch-items", "Drop files in ./inbox/ → auto-detected → Parquet → queryable in <15s" }
+                }
+            }
+
+            div { class: "panel-section",
+                h3 { "PostgreSQL Import" }
+                div { class: "form-row",
+                    label { "Host" }
+                    input { value: "{pg_host}", oninput: move |e| pg_host.set(e.value()) }
+                }
+                div { class: "form-row",
+                    label { "Database" }
+                    input { value: "{pg_db}", oninput: move |e| pg_db.set(e.value()) }
+                }
+                button { class: "btn", disabled: *pg_loading.read(), onclick: list_tables, "List Tables" }
+
+                if let Some(tables) = pg_tables.read().as_ref() {
+                    div { class: "table-list",
+                        for table in tables.iter() {
+                            {
+                                let t = table.clone();
+                                let t2 = table.clone();
+                                rsx! {
+                                    div { class: "table-item",
+                                        span { "{t}" }
+                                        button { class: "btn btn-sm", onclick: move |_| import_table(t2.clone()), "Import" }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if let Some(result) = pg_result.read().as_ref() {
+                    match result {
+                        Ok(v) => rsx! {
+                            div { class: "result-box",
+                                pre { {serde_json::to_string_pretty(v).unwrap_or_default()} }
+                            }
+                        },
+                        Err(e) => rsx! { div { class: "error", "{e}" } },
+                    }
+                }
+            }
+        }
+    }
+}
+
 // === STATUS — System health ===

 #[component]
@@ -633,6 +1031,9 @@ fn ResultsTable(response: QueryResponse) -> Element {
        if response.row_count == 0 {
            div { class: "empty-sm", "no rows returned" }
        } else if let Some(rows) = rows {
+            if rows.len() > 500 {
+                div { class: "results-info", "Showing first 500 of {response.row_count} rows (use SQL tab with LIMIT for larger)" }
+            }
            div { class: "table-wrap",
                table {
                    thead {
@@ -643,7 +1044,7 @@ fn ResultsTable(response: QueryResponse) -> Element {
                        }
                    }
                    tbody {
-                        for row in rows.iter() {
+                        for row in rows.iter().take(500) {
                            tr {
                                for col in response.columns.iter() {
                                    td { {format_cell(row.get(&col.name))} }
@@ -657,6 +1058,49 @@ fn ResultsTable(response: QueryResponse) -> Element {
    }
 }

+/// Clean AI-generated SQL: extract only the SQL query, strip everything else.
+fn clean_sql(raw: &str) -> String {
+    let s = raw.trim();
+
+    // Strategy 1: If there's a ```sql...``` block, extract just that
+    if let Some(start) = s.find("```sql") {
+        let after = &s[start + 6..];
+        if let Some(end) = after.find("```") {
+            return after[..end].trim().to_string();
+        }
+    }
+    if let Some(start) = s.find("```") {
+        let after = &s[start + 3..];
+        if let Some(end) = after.find("```") {
+            let inner = after[..end].trim();
+            // Skip leading "sql" keyword
+            let inner = inner.strip_prefix("sql").map(|s| s.trim_start()).unwrap_or(inner);
+            return inner.to_string();
+        }
+    }
+
+    // Strategy 2: Find the first SELECT/WITH/INSERT/UPDATE/DELETE statement
+    let upper = s.to_uppercase();
+    for keyword in &["SELECT", "WITH", "INSERT", "UPDATE", "DELETE"] {
+        if let Some(pos) = upper.find(keyword) {
+            let sql_part = &s[pos..];
+            // Take up to the first semicolon (or end)
+            let end = sql_part.find(';').map(|p| p + 1).unwrap_or(sql_part.len());
+            return sql_part[..end].trim().to_string();
+        }
+    }
+
+    // Strategy 3: Strip leading "sql" and clean up
+    let mut result = s.to_string();
+    let lines: Vec<&str> = result.lines().collect();
+    if let Some(first) = lines.first() {
+        if first.trim().eq_ignore_ascii_case("sql") {
+            result = lines[1..].join("\n").trim().to_string();
+        }
+    }
+    result
+}
+
 fn format_cell(val: Option<&serde_json::Value>) -> String {
    match val {
        None | Some(serde_json::Value::Null) => "—".to_string(),
@@ -0,0 +1,20 @@
+[package]
+name = "vectord"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+shared = { path = "../shared" }
+storaged = { path = "../storaged" }
+aibridge = { path = "../aibridge" }
+tokio = { workspace = true }
+axum = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tracing = { workspace = true }
+bytes = { workspace = true }
+object_store = { workspace = true }
+parquet = { workspace = true }
+arrow = { workspace = true }
+chrono = { workspace = true }
+instant-distance = { workspace = true }
@@ -0,0 +1,132 @@
+/// Text chunking strategies for embedding.
+/// Chunks need to be small enough for the embedding model (typically <512 tokens)
+/// but large enough to carry meaning.
+
+/// A chunk of text with metadata pointing back to its source.
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct TextChunk {
+    /// Source identifier (dataset name, filename, etc.)
+    pub source: String,
+    /// Row or document ID within the source
+    pub doc_id: String,
+    /// Chunk index within the document (0, 1, 2, ...)
+    pub chunk_idx: u32,
+    /// The actual text content
+    pub text: String,
+}
+
+/// Split text into overlapping chunks.
+/// - `chunk_size`: target characters per chunk (not tokens — chars are a good proxy)
+/// - `overlap`: characters of overlap between consecutive chunks
+pub fn chunk_text(
+    text: &str,
+    source: &str,
+    doc_id: &str,
+    chunk_size: usize,
+    overlap: usize,
+) -> Vec<TextChunk> {
+    let text = text.trim();
+    if text.is_empty() {
+        return vec![];
+    }
+
+    // Short text — single chunk
+    if text.len() <= chunk_size {
+        return vec![TextChunk {
+            source: source.to_string(),
+            doc_id: doc_id.to_string(),
+            chunk_idx: 0,
+            text: text.to_string(),
+        }];
+    }
+
+    // Work with char indices, not byte indices, to handle Unicode safely
+    let chars: Vec<char> = text.chars().collect();
+    let char_len = chars.len();
+    let mut chunks = Vec::new();
+    let mut start = 0usize; // char index
+    let mut idx = 0u32;
+
+    while start < char_len {
+        let end = (start + chunk_size).min(char_len);
+        let chunk_str: String = chars[start..end].iter().collect();
+
+        let actual_end = if end < char_len {
+            // Look for last sentence boundary
+            if let Some(pos) = chunk_str.rfind(". ") {
+                start + chunk_str[..pos + 2].chars().count()
+            } else if let Some(pos) = chunk_str.rfind('\n') {
+                start + chunk_str[..pos + 1].chars().count()
+            } else if let Some(pos) = chunk_str.rfind(' ') {
+                start + chunk_str[..pos + 1].chars().count()
+            } else {
+                end
+            }
+        } else {
+            end
+        };
+
+        let chunk: String = chars[start..actual_end].iter().collect();
+        let trimmed = chunk.trim();
+        if !trimmed.is_empty() {
+            chunks.push(TextChunk {
+                source: source.to_string(),
+                doc_id: doc_id.to_string(),
+                chunk_idx: idx,
+                text: trimmed.to_string(),
+            });
+            idx += 1;
+        }
+
+        if actual_end >= char_len { break; }
+        start = if actual_end > overlap { actual_end - overlap } else { actual_end };
+    }
+
+    chunks
+}
+
+/// Chunk a dataset's text column. Returns all chunks from all rows.
+pub fn chunk_column(
+    source: &str,
+    doc_ids: &[String],
+    texts: &[String],
+    chunk_size: usize,
+    overlap: usize,
+) -> Vec<TextChunk> {
+    let mut all_chunks = Vec::new();
+    for (doc_id, text) in doc_ids.iter().zip(texts.iter()) {
+        let chunks = chunk_text(text, source, doc_id, chunk_size, overlap);
+        all_chunks.extend(chunks);
+    }
+    all_chunks
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn short_text_single_chunk() {
+        let chunks = chunk_text("Hello world", "test", "1", 500, 50);
+        assert_eq!(chunks.len(), 1);
+        assert_eq!(chunks[0].text, "Hello world");
+    }
+
+    #[test]
+    fn long_text_multiple_chunks() {
+        let text = "First sentence. Second sentence. Third sentence. Fourth sentence. Fifth sentence. Sixth sentence. Seventh sentence. Eighth sentence.";
+        let chunks = chunk_text(text, "test", "1", 50, 10);
+        assert!(chunks.len() > 1);
+        // All text should be covered
+        for chunk in &chunks {
+            assert!(!chunk.text.is_empty());
+        }
+    }
+
+    #[test]
+    fn chunk_preserves_source() {
+        let chunks = chunk_text("Some text here", "candidates", "CAND-001", 500, 50);
+        assert_eq!(chunks[0].source, "candidates");
+        assert_eq!(chunks[0].doc_id, "CAND-001");
+    }
+}
@@ -0,0 +1,168 @@
+/// HNSW (Hierarchical Navigable Small World) index for fast approximate nearest neighbor search.
+/// Wraps instant-distance to provide <50ms search over 100K+ vectors.
+/// Falls back to brute-force for small datasets.
+
+use instant_distance::{Builder, HnswMap, Search, Point};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use crate::store::StoredEmbedding;
+
+/// A vector point for HNSW — wraps f32 slice with cosine distance.
+#[derive(Clone)]
+struct VecPoint(Vec<f32>);
+
+impl Point for VecPoint {
+    fn distance(&self, other: &Self) -> f32 {
+        // Cosine distance = 1 - cosine_similarity
+        let dot: f32 = self.0.iter().zip(other.0.iter()).map(|(a, b)| a * b).sum();
+        let norm_a: f32 = self.0.iter().map(|x| x * x).sum::<f32>().sqrt();
+        let norm_b: f32 = other.0.iter().map(|x| x * x).sum::<f32>().sqrt();
+        if norm_a == 0.0 || norm_b == 0.0 {
+            return 1.0;
+        }
+        1.0 - (dot / (norm_a * norm_b))
+    }
+}
+
+/// HNSW search result.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct HnswResult {
+    pub source: String,
+    pub doc_id: String,
+    pub chunk_idx: u32,
+    pub chunk_text: String,
+    pub score: f32,  // cosine similarity (1.0 = identical)
+}
+
+/// An HNSW index built from stored embeddings.
+pub struct HnswIndex {
+    map: HnswMap<VecPoint, usize>, // value is index into metadata vec
+    metadata: Vec<EmbeddingMeta>,
+}
+
+#[derive(Clone)]
+struct EmbeddingMeta {
+    source: String,
+    doc_id: String,
+    chunk_idx: u32,
+    chunk_text: String,
+}
+
+/// Shared, thread-safe index store.
+#[derive(Clone)]
+pub struct HnswStore {
+    indexes: Arc<RwLock<std::collections::HashMap<String, Arc<HnswIndex>>>>,
+}
+
+impl HnswStore {
+    pub fn new() -> Self {
+        Self {
+            indexes: Arc::new(RwLock::new(std::collections::HashMap::new())),
+        }
+    }
+
+    /// Build an HNSW index from stored embeddings.
+    pub async fn build_index(
+        &self,
+        index_name: &str,
+        embeddings: Vec<StoredEmbedding>,
+    ) -> Result<BuildStats, String> {
+        let n = embeddings.len();
+        if n == 0 {
+            return Err("no embeddings to index".into());
+        }
+
+        tracing::info!("building HNSW index '{}' from {} vectors", index_name, n);
+        let start = std::time::Instant::now();
+
+        // Separate points and metadata
+        let mut points = Vec::with_capacity(n);
+        let mut metadata = Vec::with_capacity(n);
+        let mut values = Vec::with_capacity(n);
+
+        for (i, emb) in embeddings.into_iter().enumerate() {
+            points.push(VecPoint(emb.vector));
+            metadata.push(EmbeddingMeta {
+                source: emb.source,
+                doc_id: emb.doc_id,
+                chunk_idx: emb.chunk_idx,
+                chunk_text: emb.chunk_text,
+            });
+            values.push(i);
+        }
+
+        // Build HNSW — this is the expensive part
+        let map = Builder::default()
+            .ef_construction(40)   // balanced for 100K scale
+            .ef_search(50)         // fast search with good recall
+            .build(points, values);
+
+        let build_time = start.elapsed().as_secs_f32();
+        tracing::info!("HNSW index '{}' built: {} vectors in {:.1}s", index_name, n, build_time);
+
+        let index = Arc::new(HnswIndex { map, metadata });
+        self.indexes.write().await.insert(index_name.to_string(), index);
+
+        Ok(BuildStats {
+            index_name: index_name.to_string(),
+            vectors: n,
+            build_time_secs: build_time,
+        })
+    }
+
+    /// Search an HNSW index. Returns approximate nearest neighbors.
+    pub async fn search(
+        &self,
+        index_name: &str,
+        query: &[f32],
+        top_k: usize,
+    ) -> Result<Vec<HnswResult>, String> {
+        let indexes = self.indexes.read().await;
+        let index = indexes.get(index_name)
+            .ok_or_else(|| format!("HNSW index not found: {index_name}"))?;
+
+        let query_point = VecPoint(query.to_vec());
+        let mut search = Search::default();
+        let results = index.map.search(&query_point, &mut search);
+
+        let mut out = Vec::with_capacity(top_k);
+        for item in results.take(top_k) {
+            let meta_idx = *item.value;
+            let meta = &index.metadata[meta_idx];
+            // Convert distance back to similarity
+            let similarity = 1.0 - item.distance;
+            out.push(HnswResult {
+                source: meta.source.clone(),
+                doc_id: meta.doc_id.clone(),
+                chunk_idx: meta.chunk_idx,
+                chunk_text: meta.chunk_text.clone(),
+                score: similarity,
+            });
+        }
+
+        Ok(out)
+    }
+
+    /// Check if an index exists.
+    pub async fn has_index(&self, name: &str) -> bool {
+        self.indexes.read().await.contains_key(name)
+    }
+
+    /// List all loaded indexes.
+    pub async fn list(&self) -> Vec<String> {
+        self.indexes.read().await.keys().cloned().collect()
+    }
+
+    /// Drop an index from memory.
+    pub async fn drop(&self, name: &str) -> bool {
+        self.indexes.write().await.remove(name).is_some()
+    }
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct BuildStats {
+    pub index_name: String,
+    pub vectors: usize,
+    pub build_time_secs: f32,
+}
@@ -0,0 +1,112 @@
+/// Vector index registry — tracks all indexes with model versioning.
+/// Each index knows which model created it, enabling:
+/// - Multi-version indexes (same data, different models, coexist)
+/// - Incremental re-embed (only new/changed docs on model upgrade)
+/// - A/B search comparison between model versions
+
+use chrono::{DateTime, Utc};
+use object_store::ObjectStore;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use storaged::ops;
+
+/// Metadata for a vector index.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct IndexMeta {
+    pub index_name: String,
+    pub source: String,           // dataset this was built from
+    pub model_name: String,       // "nomic-embed-text"
+    pub model_version: String,    // "latest" or specific version
+    pub dimensions: u32,          // 768
+    pub chunk_count: usize,
+    pub doc_count: usize,
+    pub chunk_size: usize,
+    pub overlap: usize,
+    pub storage_key: String,      // "vectors/resumes_v1_nomic.parquet"
+    pub created_at: DateTime<Utc>,
+    pub build_time_secs: f32,
+    pub chunks_per_sec: f32,
+}
+
+/// Registry of all vector indexes.
+#[derive(Clone)]
+pub struct IndexRegistry {
+    indexes: Arc<RwLock<HashMap<String, IndexMeta>>>,
+    store: Arc<dyn ObjectStore>,
+}
+
+impl IndexRegistry {
+    pub fn new(store: Arc<dyn ObjectStore>) -> Self {
+        Self {
+            indexes: Arc::new(RwLock::new(HashMap::new())),
+            store,
+        }
+    }
+
+    /// Rebuild from persisted index metadata on startup.
+    pub async fn rebuild(&self) -> Result<usize, String> {
+        let keys = ops::list(&self.store, Some("vectors/meta/")).await?;
+        let mut reg = self.indexes.write().await;
+        reg.clear();
+
+        for key in &keys {
+            if !key.ends_with(".json") { continue; }
+            let data = ops::get(&self.store, key).await?;
+            match serde_json::from_slice::<IndexMeta>(&data) {
+                Ok(meta) => { reg.insert(meta.index_name.clone(), meta); }
+                Err(e) => tracing::warn!("failed to load index meta {key}: {e}"),
+            }
+        }
+
+        let count = reg.len();
+        if count > 0 {
+            tracing::info!("loaded {count} vector index metadata entries");
+        }
+        Ok(count)
+    }
+
+    /// Register a new index.
+    pub async fn register(&self, meta: IndexMeta) -> Result<(), String> {
+        let key = format!("vectors/meta/{}.json", meta.index_name);
+        let json = serde_json::to_vec_pretty(&meta).map_err(|e| e.to_string())?;
+        ops::put(&self.store, &key, json.into()).await?;
+        self.indexes.write().await.insert(meta.index_name.clone(), meta);
+        Ok(())
+    }
+
+    /// Get metadata for an index.
+    pub async fn get(&self, index_name: &str) -> Option<IndexMeta> {
+        self.indexes.read().await.get(index_name).cloned()
+    }
+
+    /// List all indexes, optionally filtered by source or model.
+    pub async fn list(&self, source: Option<&str>, model: Option<&str>) -> Vec<IndexMeta> {
+        self.indexes.read().await.values()
+            .filter(|m| source.map_or(true, |s| m.source == s))
+            .filter(|m| model.map_or(true, |mo| m.model_name == mo))
+            .cloned()
+            .collect()
+    }
+
+    /// Find all versions of an index for a given source dataset.
+    /// Returns indexes sorted by creation time (newest first).
+    pub async fn versions_for_source(&self, source: &str) -> Vec<IndexMeta> {
+        let mut versions: Vec<IndexMeta> = self.indexes.read().await.values()
+            .filter(|m| m.source == source)
+            .cloned()
+            .collect();
+        versions.sort_by(|a, b| b.created_at.cmp(&a.created_at));
+        versions
+    }
+
+    /// Delete an index (metadata only — vector Parquet stays for safety).
+    pub async fn delete(&self, index_name: &str) -> Result<(), String> {
+        let key = format!("vectors/meta/{index_name}.json");
+        ops::delete(&self.store, &key).await?;
+        self.indexes.write().await.remove(index_name);
+        Ok(())
+    }
+}
@@ -0,0 +1,112 @@
+/// Background job system for long-running embedding tasks.
+/// POST /vectors/index returns a job_id immediately.
+/// GET /vectors/jobs/{id} returns progress.
+/// Embedding runs in background via tokio::spawn.
+
+use serde::Serialize;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum JobStatus {
+    Running,
+    Completed,
+    Failed,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct Job {
+    pub id: String,
+    pub status: JobStatus,
+    pub index_name: String,
+    pub total_chunks: usize,
+    pub embedded_chunks: usize,
+    pub progress_pct: f32,
+    pub storage_key: Option<String>,
+    pub error: Option<String>,
+    pub started_at: String,
+    pub completed_at: Option<String>,
+    pub chunks_per_sec: f32,
+}
+
+/// Shared progress tracker that background tasks update.
+#[derive(Clone)]
+pub struct JobTracker {
+    jobs: Arc<RwLock<HashMap<String, Job>>>,
+}
+
+impl JobTracker {
+    pub fn new() -> Self {
+        Self {
+            jobs: Arc::new(RwLock::new(HashMap::new())),
+        }
+    }
+
+    /// Register a new job. Returns the job ID.
+    pub async fn create(&self, index_name: &str, total_chunks: usize) -> String {
+        let id = format!("job-{}", chrono::Utc::now().timestamp_millis());
+        let job = Job {
+            id: id.clone(),
+            status: JobStatus::Running,
+            index_name: index_name.to_string(),
+            total_chunks,
+            embedded_chunks: 0,
+            progress_pct: 0.0,
+            storage_key: None,
+            error: None,
+            started_at: chrono::Utc::now().to_rfc3339(),
+            completed_at: None,
+            chunks_per_sec: 0.0,
+        };
+        self.jobs.write().await.insert(id.clone(), job);
+        id
+    }
+
+    /// Update progress.
+    pub async fn update_progress(&self, id: &str, embedded: usize, rate: f32) {
+        let mut jobs = self.jobs.write().await;
+        if let Some(job) = jobs.get_mut(id) {
+            job.embedded_chunks = embedded;
+            job.progress_pct = if job.total_chunks > 0 {
+                (embedded as f32 / job.total_chunks as f32) * 100.0
+            } else {
+                0.0
+            };
+            job.chunks_per_sec = rate;
+        }
+    }
+
+    /// Mark job as completed.
+    pub async fn complete(&self, id: &str, storage_key: String) {
+        let mut jobs = self.jobs.write().await;
+        if let Some(job) = jobs.get_mut(id) {
+            job.status = JobStatus::Completed;
+            job.embedded_chunks = job.total_chunks;
+            job.progress_pct = 100.0;
+            job.storage_key = Some(storage_key);
+            job.completed_at = Some(chrono::Utc::now().to_rfc3339());
+        }
+    }
+
+    /// Mark job as failed.
+    pub async fn fail(&self, id: &str, error: String) {
+        let mut jobs = self.jobs.write().await;
+        if let Some(job) = jobs.get_mut(id) {
+            job.status = JobStatus::Failed;
+            job.error = Some(error);
+            job.completed_at = Some(chrono::Utc::now().to_rfc3339());
+        }
+    }
+
+    /// Get job status.
+    pub async fn get(&self, id: &str) -> Option<Job> {
+        self.jobs.read().await.get(id).cloned()
+    }
+
+    /// List all jobs.
+    pub async fn list(&self) -> Vec<Job> {
+        self.jobs.read().await.values().cloned().collect()
+    }
+}
@@ -0,0 +1,9 @@
+pub mod chunker;
+pub mod hnsw;
+pub mod index_registry;
+pub mod jobs;
+pub mod store;
+pub mod search;
+pub mod rag;
+pub mod supervisor;
+pub mod service;
@@ -0,0 +1,84 @@
+/// RAG pipeline: question → embed → search → retrieve → generate answer.
+
+use object_store::ObjectStore;
+use std::sync::Arc;
+
+use aibridge::client::{AiClient, EmbedRequest, GenerateRequest};
+use crate::search::{self, SearchResult};
+use crate::store;
+
+/// Full RAG answer with provenance.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct RagResponse {
+    pub answer: String,
+    pub model: String,
+    pub sources: Vec<SearchResult>,
+    pub tokens_generated: Option<u64>,
+}
+
+/// Execute full RAG: embed question → search index → retrieve context → generate answer.
+pub async fn query(
+    question: &str,
+    index_name: &str,
+    top_k: usize,
+    object_store: &Arc<dyn ObjectStore>,
+    ai_client: &AiClient,
+) -> Result<RagResponse, String> {
+    // 1. Embed the question
+    tracing::info!("RAG: embedding question");
+    let embed_resp = ai_client.embed(EmbedRequest {
+        texts: vec![question.to_string()],
+        model: None,
+    }).await?;
+
+    if embed_resp.embeddings.is_empty() {
+        return Err("no embedding returned for question".into());
+    }
+
+    let query_vec: Vec<f32> = embed_resp.embeddings[0].iter().map(|&x| x as f32).collect();
+
+    // 2. Load index and search
+    tracing::info!("RAG: searching index '{index_name}'");
+    let embeddings = store::load_embeddings(object_store, index_name).await?;
+    let results = search::search(&query_vec, &embeddings, top_k);
+
+    if results.is_empty() {
+        return Ok(RagResponse {
+            answer: "No relevant information found.".into(),
+            model: String::new(),
+            sources: vec![],
+            tokens_generated: None,
+        });
+    }
+
+    // 3. Build context from retrieved chunks
+    let context: String = results.iter().enumerate().map(|(i, r)| {
+        format!("[{}] (source: {}, doc: {}) {}", i + 1, r.source, r.doc_id, r.chunk_text)
+    }).collect::<Vec<_>>().join("\n\n");
+
+    // 4. Generate answer
+    tracing::info!("RAG: generating answer from {} chunks", results.len());
+    let prompt = format!(
+        "You are a helpful assistant answering questions based on retrieved documents from a data system.\n\n\
+        Use ONLY the following context to answer. If the context doesn't contain enough information, say so.\n\
+        Cite sources by their number [1], [2], etc.\n\n\
+        Context:\n{context}\n\n\
+        Question: {question}\n\n\
+        Answer:"
+    );
+
+    let gen_resp = ai_client.generate(GenerateRequest {
+        prompt,
+        model: None,
+        system: None,
+        temperature: Some(0.2),
+        max_tokens: Some(512),
+    }).await?;
+
+    Ok(RagResponse {
+        answer: gen_resp.text.trim().to_string(),
+        model: gen_resp.model,
+        sources: results,
+        tokens_generated: gen_resp.tokens_generated,
+    })
+}
@@ -0,0 +1,107 @@
+/// Brute-force vector search with cosine similarity.
+/// Works well up to ~100K vectors. HNSW index would go here for larger scale.
+
+use crate::store::StoredEmbedding;
+
+/// A search result with score.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct SearchResult {
+    pub source: String,
+    pub doc_id: String,
+    pub chunk_idx: u32,
+    pub chunk_text: String,
+    pub score: f32,
+}
+
+/// Search embeddings by cosine similarity. Returns top_k results.
+pub fn search(
+    query_vector: &[f32],
+    embeddings: &[StoredEmbedding],
+    top_k: usize,
+) -> Vec<SearchResult> {
+    let query_norm = norm(query_vector);
+    if query_norm == 0.0 {
+        return vec![];
+    }
+
+    let mut scored: Vec<SearchResult> = embeddings.iter().map(|emb| {
+        let score = cosine_similarity(query_vector, &emb.vector, query_norm);
+        SearchResult {
+            source: emb.source.clone(),
+            doc_id: emb.doc_id.clone(),
+            chunk_idx: emb.chunk_idx,
+            chunk_text: emb.chunk_text.clone(),
+            score,
+        }
+    }).collect();
+
+    // Sort descending by score
+    scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
+    scored.truncate(top_k);
+    scored
+}
+
+fn cosine_similarity(a: &[f32], b: &[f32], a_norm: f32) -> f32 {
+    if a.len() != b.len() {
+        return 0.0;
+    }
+    let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
+    let b_norm = norm(b);
+    if b_norm == 0.0 {
+        return 0.0;
+    }
+    dot / (a_norm * b_norm)
+}
+
+fn norm(v: &[f32]) -> f32 {
+    v.iter().map(|x| x * x).sum::<f32>().sqrt()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn identical_vectors_score_1() {
+        let v = vec![1.0, 2.0, 3.0];
+        let emb = StoredEmbedding {
+            source: "test".into(),
+            doc_id: "1".into(),
+            chunk_idx: 0,
+            chunk_text: "hello".into(),
+            vector: v.clone(),
+        };
+        let results = search(&v, &[emb], 1);
+        assert!((results[0].score - 1.0).abs() < 0.001);
+    }
+
+    #[test]
+    fn orthogonal_vectors_score_0() {
+        let q = vec![1.0, 0.0];
+        let emb = StoredEmbedding {
+            source: "test".into(),
+            doc_id: "1".into(),
+            chunk_idx: 0,
+            chunk_text: "hello".into(),
+            vector: vec![0.0, 1.0],
+        };
+        let results = search(&q, &[emb], 1);
+        assert!(results[0].score.abs() < 0.001);
+    }
+
+    #[test]
+    fn returns_top_k() {
+        let q = vec![1.0, 0.0, 0.0];
+        let embs: Vec<StoredEmbedding> = (0..10).map(|i| StoredEmbedding {
+            source: "test".into(),
+            doc_id: format!("{i}"),
+            chunk_idx: 0,
+            chunk_text: format!("doc {i}"),
+            vector: vec![1.0 - i as f32 * 0.1, i as f32 * 0.1, 0.0],
+        }).collect();
+        let results = search(&q, &embs, 3);
+        assert_eq!(results.len(), 3);
+        assert!(results[0].score >= results[1].score);
+        assert!(results[1].score >= results[2].score);
+    }
+}
@@ -0,0 +1,374 @@
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use object_store::ObjectStore;
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+use aibridge::client::{AiClient, EmbedRequest};
+use crate::{chunker, hnsw, index_registry, jobs, rag, search, store, supervisor};
+
+#[derive(Clone)]
+pub struct VectorState {
+    pub store: Arc<dyn ObjectStore>,
+    pub ai_client: AiClient,
+    pub job_tracker: jobs::JobTracker,
+    pub index_registry: index_registry::IndexRegistry,
+    pub hnsw_store: hnsw::HnswStore,
+}
+
+pub fn router(state: VectorState) -> Router {
+    Router::new()
+        .route("/health", get(health))
+        .route("/index", post(create_index))
+        .route("/indexes", get(list_indexes))
+        .route("/indexes/{name}", get(get_index_meta))
+        .route("/jobs", get(list_jobs))
+        .route("/jobs/{id}", get(get_job))
+        .route("/search", post(search_index))
+        .route("/rag", post(rag_query))
+        .route("/hnsw/build", post(build_hnsw))
+        .route("/hnsw/search", post(search_hnsw))
+        .route("/hnsw/list", get(list_hnsw))
+        .with_state(state)
+}
+
+async fn health() -> &'static str {
+    "vectord ok"
+}
+
+// --- Background Index Creation ---
+
+#[derive(Deserialize)]
+struct CreateIndexRequest {
+    index_name: String,
+    source: String,
+    documents: Vec<DocInput>,
+    chunk_size: Option<usize>,
+    overlap: Option<usize>,
+}
+
+#[derive(Deserialize)]
+struct DocInput {
+    id: String,
+    text: String,
+}
+
+#[derive(Serialize)]
+struct CreateIndexResponse {
+    job_id: String,
+    index_name: String,
+    documents: usize,
+    chunks: usize,
+    message: String,
+}
+
+async fn create_index(
+    State(state): State<VectorState>,
+    Json(req): Json<CreateIndexRequest>,
+) -> impl IntoResponse {
+    let chunk_size = req.chunk_size.unwrap_or(500);
+    let overlap = req.overlap.unwrap_or(50);
+
+    // Chunk synchronously (fast)
+    let doc_ids: Vec<String> = req.documents.iter().map(|d| d.id.clone()).collect();
+    let texts: Vec<String> = req.documents.iter().map(|d| d.text.clone()).collect();
+    let chunks = chunker::chunk_column(&req.source, &doc_ids, &texts, chunk_size, overlap);
+
+    if chunks.is_empty() {
+        return Err((StatusCode::BAD_REQUEST, "no text to index".to_string()));
+    }
+
+    let n_docs = req.documents.len();
+    let n_chunks = chunks.len();
+    let index_name = req.index_name.clone();
+
+    // Create job and return immediately
+    let job_id = state.job_tracker.create(&index_name, n_chunks).await;
+    tracing::info!("job {job_id}: indexing '{}' — {} docs → {} chunks (background)", index_name, n_docs, n_chunks);
+
+    // Spawn supervised dual-pipeline embedding
+    let tracker = state.job_tracker.clone();
+    let ai_client = state.ai_client.clone();
+    let obj_store = state.store.clone();
+    let registry = state.index_registry.clone();
+    let jid = job_id.clone();
+    let source_name = req.source.clone();
+    let idx_name = req.index_name.clone();
+
+    tokio::spawn(async move {
+        let start_time = std::time::Instant::now();
+        let config = supervisor::SupervisorConfig::default();
+        let result = supervisor::run_supervised(
+            &jid, &idx_name, chunks, &ai_client, &obj_store, &tracker, config,
+        ).await;
+        match result {
+            Ok(key) => {
+                let elapsed = start_time.elapsed().as_secs_f32();
+                let rate = if elapsed > 0.0 { n_chunks as f32 / elapsed } else { 0.0 };
+
+                // Register index metadata with model version info
+                let meta = index_registry::IndexMeta {
+                    index_name: idx_name.clone(),
+                    source: source_name,
+                    model_name: "nomic-embed-text".to_string(), // from sidecar config
+                    model_version: "latest".to_string(),
+                    dimensions: 768,
+                    chunk_count: n_chunks,
+                    doc_count: n_docs,
+                    chunk_size: chunk_size,
+                    overlap: overlap,
+                    storage_key: key.clone(),
+                    created_at: chrono::Utc::now(),
+                    build_time_secs: elapsed,
+                    chunks_per_sec: rate,
+                };
+                let _ = registry.register(meta).await;
+
+                tracker.complete(&jid, key).await;
+                tracing::info!("job {jid}: completed — {n_chunks} chunks in {elapsed:.0}s ({rate:.0}/sec)");
+            }
+            Err(e) => {
+                tracker.fail(&jid, e.clone()).await;
+                tracing::error!("job {jid}: failed — {e}");
+            }
+        }
+    });
+
+    Ok((StatusCode::ACCEPTED, Json(CreateIndexResponse {
+        job_id,
+        index_name: req.index_name,
+        documents: n_docs,
+        chunks: n_chunks,
+        message: format!("embedding {} chunks in background — poll /vectors/jobs/{{id}} for progress", n_chunks),
+    })))
+}
+
+// --- Index Registry ---
+
+#[derive(Deserialize)]
+struct IndexListQuery {
+    source: Option<String>,
+    model: Option<String>,
+}
+
+async fn list_indexes(
+    State(state): State<VectorState>,
+    Query(q): Query<IndexListQuery>,
+) -> impl IntoResponse {
+    let indexes = state.index_registry.list(q.source.as_deref(), q.model.as_deref()).await;
+    Json(indexes)
+}
+
+async fn get_index_meta(
+    State(state): State<VectorState>,
+    Path(name): Path<String>,
+) -> impl IntoResponse {
+    match state.index_registry.get(&name).await {
+        Some(meta) => Ok(Json(meta)),
+        None => Err((StatusCode::NOT_FOUND, format!("index not found: {name}"))),
+    }
+}
+
+// --- unused legacy function below, kept for reference ---
+
+#[allow(dead_code)]
+/// Legacy single-pipeline embedding (replaced by supervisor).
+async fn _run_embedding_job_legacy(
+    job_id: &str,
+    index_name: &str,
+    chunks: &[chunker::TextChunk],
+    ai_client: &AiClient,
+    store: &Arc<dyn ObjectStore>,
+    tracker: &jobs::JobTracker,
+) -> Result<String, String> {
+    let batch_size = 32;
+    let mut all_vectors: Vec<Vec<f64>> = Vec::new();
+    let start = std::time::Instant::now();
+
+    for (i, batch) in chunks.chunks(batch_size).enumerate() {
+        let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
+
+        let embed_resp = ai_client.embed(EmbedRequest {
+            texts,
+            model: None,
+        }).await.map_err(|e| format!("embed batch {} error: {e}", i))?;
+
+        all_vectors.extend(embed_resp.embeddings);
+
+        // Update progress
+        let elapsed = start.elapsed().as_secs_f32();
+        let rate = if elapsed > 0.0 { all_vectors.len() as f32 / elapsed } else { 0.0 };
+        tracker.update_progress(job_id, all_vectors.len(), rate).await;
+
+        // Log every 100 batches
+        if (i + 1) % 100 == 0 {
+            let pct = (all_vectors.len() as f32 / chunks.len() as f32) * 100.0;
+            let eta = if rate > 0.0 { (chunks.len() - all_vectors.len()) as f32 / rate } else { 0.0 };
+            tracing::info!("job {job_id}: {}/{} chunks ({pct:.0}%), {rate:.0}/sec, ETA {eta:.0}s",
+                all_vectors.len(), chunks.len());
+        }
+    }
+
+    // Store
+    let key = store::store_embeddings(store, index_name, chunks, &all_vectors).await?;
+    Ok(key)
+}
+
+// --- Job Status ---
+
+async fn list_jobs(State(state): State<VectorState>) -> impl IntoResponse {
+    let jobs = state.job_tracker.list().await;
+    Json(jobs)
+}
+
+async fn get_job(
+    State(state): State<VectorState>,
+    Path(id): Path<String>,
+) -> impl IntoResponse {
+    match state.job_tracker.get(&id).await {
+        Some(job) => Ok(Json(job)),
+        None => Err((StatusCode::NOT_FOUND, format!("job not found: {id}"))),
+    }
+}
+
+// --- Search ---
+
+#[derive(Deserialize)]
+struct SearchRequest {
+    index_name: String,
+    query: String,
+    top_k: Option<usize>,
+}
+
+#[derive(Serialize)]
+struct SearchResponse {
+    results: Vec<search::SearchResult>,
+    query: String,
+}
+
+async fn search_index(
+    State(state): State<VectorState>,
+    Json(req): Json<SearchRequest>,
+) -> impl IntoResponse {
+    let top_k = req.top_k.unwrap_or(5);
+
+    let embed_resp = state.ai_client.embed(EmbedRequest {
+        texts: vec![req.query.clone()],
+        model: None,
+    }).await.map_err(|e| (StatusCode::BAD_GATEWAY, format!("embed error: {e}")))?;
+
+    if embed_resp.embeddings.is_empty() {
+        return Err((StatusCode::BAD_GATEWAY, "no embedding returned".to_string()));
+    }
+
+    let query_vec: Vec<f32> = embed_resp.embeddings[0].iter().map(|&x| x as f32).collect();
+
+    let embeddings = store::load_embeddings(&state.store, &req.index_name)
+        .await
+        .map_err(|e| (StatusCode::NOT_FOUND, format!("index not found: {e}")))?;
+
+    let results = search::search(&query_vec, &embeddings, top_k);
+
+    Ok(Json(SearchResponse {
+        results,
+        query: req.query,
+    }))
+}
+
+// --- RAG ---
+
+#[derive(Deserialize)]
+struct RagRequest {
+    index_name: String,
+    question: String,
+    top_k: Option<usize>,
+}
+
+async fn rag_query(
+    State(state): State<VectorState>,
+    Json(req): Json<RagRequest>,
+) -> impl IntoResponse {
+    let top_k = req.top_k.unwrap_or(5);
+
+    match rag::query(&req.question, &req.index_name, top_k, &state.store, &state.ai_client).await {
+        Ok(resp) => Ok(Json(resp)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+// --- HNSW Fast Search ---
+
+#[derive(Deserialize)]
+struct BuildHnswRequest {
+    /// Name of the stored vector index to build HNSW from
+    index_name: String,
+}
+
+/// Build an HNSW index from an existing stored vector index.
+/// Loads embeddings from Parquet, builds HNSW in memory.
+async fn build_hnsw(
+    State(state): State<VectorState>,
+    Json(req): Json<BuildHnswRequest>,
+) -> impl IntoResponse {
+    tracing::info!("building HNSW for '{}'", req.index_name);
+
+    // Load embeddings from Parquet
+    let embeddings = store::load_embeddings(&state.store, &req.index_name)
+        .await
+        .map_err(|e| (StatusCode::NOT_FOUND, format!("index not found: {e}")))?;
+
+    let n = embeddings.len();
+    tracing::info!("loaded {} embeddings, building HNSW...", n);
+
+    // Build HNSW
+    match state.hnsw_store.build_index(&req.index_name, embeddings).await {
+        Ok(stats) => Ok(Json(stats)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct HnswSearchRequest {
+    index_name: String,
+    query: String,
+    top_k: Option<usize>,
+}
+
+/// Search using HNSW — approximate nearest neighbors, much faster than brute-force.
+async fn search_hnsw(
+    State(state): State<VectorState>,
+    Json(req): Json<HnswSearchRequest>,
+) -> impl IntoResponse {
+    let top_k = req.top_k.unwrap_or(5);
+
+    // Embed query
+    let embed_resp = state.ai_client.embed(EmbedRequest {
+        texts: vec![req.query.clone()],
+        model: None,
+    }).await.map_err(|e| (StatusCode::BAD_GATEWAY, format!("embed error: {e}")))?;
+
+    if embed_resp.embeddings.is_empty() {
+        return Err((StatusCode::BAD_GATEWAY, "no embedding returned".to_string()));
+    }
+
+    let query_vec: Vec<f32> = embed_resp.embeddings[0].iter().map(|&x| x as f32).collect();
+
+    // Search HNSW
+    match state.hnsw_store.search(&req.index_name, &query_vec, top_k).await {
+        Ok(results) => Ok(Json(serde_json::json!({
+            "results": results,
+            "query": req.query,
+            "method": "hnsw",
+        }))),
+        Err(e) => Err((StatusCode::NOT_FOUND, e)),
+    }
+}
+
+async fn list_hnsw(State(state): State<VectorState>) -> impl IntoResponse {
+    Json(state.hnsw_store.list().await)
+}
@@ -0,0 +1,116 @@
+/// Vector storage as Parquet files.
+/// Each embedding index is stored as: source, doc_id, chunk_idx, chunk_text, vector (binary blob).
+/// Vectors are stored as raw f32 bytes for compact storage and fast loading.
+
+use arrow::array::{ArrayRef, BinaryArray, Float32Array, Int32Array, RecordBatch, StringArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use bytes::Bytes;
+use object_store::ObjectStore;
+use std::sync::Arc;
+
+use storaged::ops;
+
+use crate::chunker::TextChunk;
+
+/// A stored embedding — chunk text + its vector.
+#[derive(Debug, Clone)]
+pub struct StoredEmbedding {
+    pub source: String,
+    pub doc_id: String,
+    pub chunk_idx: u32,
+    pub chunk_text: String,
+    pub vector: Vec<f32>,
+}
+
+/// Store embeddings as a Parquet file in object storage.
+pub async fn store_embeddings(
+    store: &Arc<dyn ObjectStore>,
+    index_name: &str,
+    chunks: &[TextChunk],
+    vectors: &[Vec<f64>], // from embedding API (f64), we store as f32
+) -> Result<String, String> {
+    if chunks.len() != vectors.len() {
+        return Err(format!("chunk count ({}) != vector count ({})", chunks.len(), vectors.len()));
+    }
+
+    let n = chunks.len();
+    let sources: Vec<&str> = chunks.iter().map(|c| c.source.as_str()).collect();
+    let doc_ids: Vec<&str> = chunks.iter().map(|c| c.doc_id.as_str()).collect();
+    let chunk_idxs: Vec<i32> = chunks.iter().map(|c| c.chunk_idx as i32).collect();
+    let texts: Vec<&str> = chunks.iter().map(|c| c.text.as_str()).collect();
+
+    // Store vectors as raw f32 bytes (compact binary blob)
+    let vector_bytes: Vec<Vec<u8>> = vectors.iter().map(|v| {
+        v.iter().map(|&x| x as f32).flat_map(|f| f.to_le_bytes()).collect()
+    }).collect();
+    let vector_refs: Vec<&[u8]> = vector_bytes.iter().map(|v| v.as_slice()).collect();
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("source", DataType::Utf8, false),
+        Field::new("doc_id", DataType::Utf8, false),
+        Field::new("chunk_idx", DataType::Int32, false),
+        Field::new("chunk_text", DataType::Utf8, false),
+        Field::new("vector", DataType::Binary, false),
+    ]));
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(StringArray::from(sources)),
+        Arc::new(StringArray::from(doc_ids)),
+        Arc::new(Int32Array::from(chunk_idxs)),
+        Arc::new(StringArray::from(texts)),
+        Arc::new(BinaryArray::from(vector_refs)),
+    ];
+
+    let batch = RecordBatch::try_new(schema, arrays)
+        .map_err(|e| format!("RecordBatch error: {e}"))?;
+
+    let parquet = shared::arrow_helpers::record_batch_to_parquet(&batch)?;
+    let key = format!("vectors/{index_name}.parquet");
+    ops::put(store, &key, parquet).await?;
+
+    tracing::info!("stored {n} embeddings in {key}");
+    Ok(key)
+}
+
+/// Load all embeddings from a vector index file.
+pub async fn load_embeddings(
+    store: &Arc<dyn ObjectStore>,
+    index_name: &str,
+) -> Result<Vec<StoredEmbedding>, String> {
+    let key = format!("vectors/{index_name}.parquet");
+    let data = ops::get(store, &key).await?;
+
+    let (_, batches) = shared::arrow_helpers::parquet_to_record_batches(&data)?;
+
+    let mut embeddings = Vec::new();
+    for batch in &batches {
+        let sources = batch.column(0).as_any().downcast_ref::<StringArray>()
+            .ok_or("source column not string")?;
+        let doc_ids = batch.column(1).as_any().downcast_ref::<StringArray>()
+            .ok_or("doc_id column not string")?;
+        let chunk_idxs = batch.column(2).as_any().downcast_ref::<Int32Array>()
+            .ok_or("chunk_idx column not int")?;
+        let texts = batch.column(3).as_any().downcast_ref::<StringArray>()
+            .ok_or("chunk_text column not string")?;
+        let vectors = batch.column(4).as_any().downcast_ref::<BinaryArray>()
+            .ok_or("vector column not binary")?;
+
+        for i in 0..batch.num_rows() {
+            let vec_bytes = vectors.value(i);
+            let vector: Vec<f32> = vec_bytes.chunks_exact(4)
+                .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
+                .collect();
+
+            embeddings.push(StoredEmbedding {
+                source: sources.value(i).to_string(),
+                doc_id: doc_ids.value(i).to_string(),
+                chunk_idx: chunk_idxs.value(i) as u32,
+                chunk_text: texts.value(i).to_string(),
+                vector,
+            });
+        }
+    }
+
+    tracing::info!("loaded {} embeddings from {key}", embeddings.len());
+    Ok(embeddings)
+}
@@ -0,0 +1,286 @@
+/// Dual-pipeline supervisor for embedding ingestion.
+/// Splits work into ranges, runs parallel pipelines, handles failures
+/// with round-robin retry and checkpointing.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use aibridge::client::{AiClient, EmbedRequest};
+use crate::chunker::TextChunk;
+use crate::jobs::JobTracker;
+use crate::store;
+use object_store::ObjectStore;
+
+/// A range of chunks to embed.
+#[derive(Debug, Clone)]
+struct ChunkRange {
+    start: usize,
+    end: usize,
+    attempts: u32,
+}
+
+/// Checkpoint: tracks which ranges are done, persisted for crash recovery.
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct Checkpoint {
+    pub job_id: String,
+    pub index_name: String,
+    pub total_chunks: usize,
+    pub completed_ranges: Vec<(usize, usize)>, // (start, end) pairs
+    pub failed_ranges: Vec<(usize, usize, String)>, // (start, end, error)
+    pub embedded_count: usize,
+}
+
+/// Pipeline status for monitoring.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct PipelineStatus {
+    pub pipeline_id: String,
+    pub current_range: Option<(usize, usize)>,
+    pub chunks_done: usize,
+    pub rate: f32,
+    pub alive: bool,
+}
+
+/// Supervisor configuration.
+pub struct SupervisorConfig {
+    pub num_pipelines: usize,   // default 2
+    pub batch_size: usize,      // embeddings per API call (default 32)
+    pub range_size: usize,      // chunks per range (default 2000)
+    pub max_retries: u32,       // per range (default 3)
+    pub checkpoint_interval: usize, // checkpoint every N chunks (default 1000)
+}
+
+impl Default for SupervisorConfig {
+    fn default() -> Self {
+        Self {
+            num_pipelines: 4,     // i9 + A4000 can handle parallel embedding
+            batch_size: 64,       // larger batches for GPU throughput
+            range_size: 2500,     // bigger ranges, fewer coordination overhead
+            max_retries: 3,
+            checkpoint_interval: 1000,
+        }
+    }
+}
+
+/// Run supervised dual-pipeline embedding.
+pub async fn run_supervised(
+    job_id: &str,
+    index_name: &str,
+    chunks: Vec<TextChunk>,
+    ai_client: &AiClient,
+    obj_store: &Arc<dyn ObjectStore>,
+    tracker: &JobTracker,
+    config: SupervisorConfig,
+) -> Result<String, String> {
+    let total = chunks.len();
+    let chunks = Arc::new(chunks);
+
+    tracing::info!("supervisor: starting {} pipelines for {} chunks (range_size={})",
+        config.num_pipelines, total, config.range_size);
+
+    // Split into ranges
+    let mut ranges: Vec<ChunkRange> = Vec::new();
+    let mut start = 0;
+    while start < total {
+        let end = (start + config.range_size).min(total);
+        ranges.push(ChunkRange { start, end, attempts: 0 });
+        start = end;
+    }
+
+    tracing::info!("supervisor: {} ranges created", ranges.len());
+
+    // Shared state
+    let work_queue: Arc<RwLock<Vec<ChunkRange>>> = Arc::new(RwLock::new(ranges));
+    let all_vectors: Arc<RwLock<HashMap<usize, Vec<Vec<f64>>>>> = Arc::new(RwLock::new(HashMap::new()));
+    let dead_letter: Arc<RwLock<Vec<(usize, usize, String)>>> = Arc::new(RwLock::new(Vec::new()));
+    let global_count: Arc<std::sync::atomic::AtomicUsize> = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let start_time = std::time::Instant::now();
+
+    // Checkpoint state
+    let checkpoint: Arc<RwLock<Checkpoint>> = Arc::new(RwLock::new(Checkpoint {
+        job_id: job_id.to_string(),
+        index_name: index_name.to_string(),
+        total_chunks: total,
+        completed_ranges: vec![],
+        failed_ranges: vec![],
+        embedded_count: 0,
+    }));
+
+    // Try to load existing checkpoint (crash recovery)
+    if let Ok(data) = storaged::ops::get(obj_store, &format!("checkpoints/{job_id}.json")).await {
+        if let Ok(saved) = serde_json::from_slice::<Checkpoint>(&data) {
+            tracing::info!("supervisor: recovered checkpoint — {}/{} already done",
+                saved.embedded_count, saved.total_chunks);
+
+            // Remove already-completed ranges from work queue
+            let mut queue = work_queue.write().await;
+            queue.retain(|r| !saved.completed_ranges.iter().any(|(s, e)| *s == r.start && *e == r.end));
+            global_count.store(saved.embedded_count, std::sync::atomic::Ordering::Relaxed);
+
+            *checkpoint.write().await = saved;
+        }
+    }
+
+    // Spawn pipeline workers
+    let mut handles = Vec::new();
+    for pipeline_id in 0..config.num_pipelines {
+        let queue = work_queue.clone();
+        let vectors = all_vectors.clone();
+        let dead = dead_letter.clone();
+        let count = global_count.clone();
+        let ckpt = checkpoint.clone();
+        let client = ai_client.clone();
+        let store = obj_store.clone();
+        let chunks = chunks.clone();
+        let tracker = tracker.clone();
+        let jid = job_id.to_string();
+        let batch_size = config.batch_size;
+        let max_retries = config.max_retries;
+        let ckpt_interval = config.checkpoint_interval;
+
+        let handle = tokio::spawn(async move {
+            let pid = format!("pipeline-{pipeline_id}");
+            tracing::info!("{pid}: started");
+
+            loop {
+                // Grab next range from queue
+                let range = {
+                    let mut q = queue.write().await;
+                    q.pop()
+                };
+
+                let range = match range {
+                    Some(r) => r,
+                    None => {
+                        tracing::info!("{pid}: no more work, shutting down");
+                        break;
+                    }
+                };
+
+                tracing::debug!("{pid}: processing range [{}, {})", range.start, range.end);
+
+                // Embed the range
+                match embed_range(&chunks[range.start..range.end], &client, batch_size).await {
+                    Ok(range_vectors) => {
+                        let n = range_vectors.len();
+                        vectors.write().await.insert(range.start, range_vectors);
+                        let prev = count.fetch_add(n, std::sync::atomic::Ordering::Relaxed);
+                        let done = prev + n;
+
+                        // Update job tracker
+                        let elapsed = start_time.elapsed().as_secs_f32();
+                        let rate = if elapsed > 0.0 { done as f32 / elapsed } else { 0.0 };
+                        tracker.update_progress(&jid, done, rate).await;
+
+                        // Update checkpoint
+                        let mut ckpt = ckpt.write().await;
+                        ckpt.completed_ranges.push((range.start, range.end));
+                        ckpt.embedded_count = done;
+
+                        // Persist checkpoint periodically
+                        if done % ckpt_interval < n {
+                            let json = serde_json::to_vec(&*ckpt).unwrap_or_default();
+                            let _ = storaged::ops::put(&store, &format!("checkpoints/{jid}.json"), json.into()).await;
+                        }
+
+                        tracing::info!("{pid}: range [{},{}) done — {done}/{} total",
+                            range.start, range.end, chunks.len());
+                    }
+                    Err(e) => {
+                        let attempt = range.attempts + 1;
+                        tracing::warn!("{pid}: range [{},{}) failed (attempt {attempt}/{max_retries}): {e}",
+                            range.start, range.end);
+
+                        if attempt < max_retries {
+                            // Push back to queue for retry (round-robin to other pipeline)
+                            queue.write().await.push(ChunkRange {
+                                start: range.start,
+                                end: range.end,
+                                attempts: attempt,
+                            });
+                        } else {
+                            // Dead letter
+                            tracing::error!("{pid}: range [{},{}) dead-lettered after {max_retries} attempts",
+                                range.start, range.end);
+                            dead.write().await.push((range.start, range.end, e));
+
+                            let mut ckpt = ckpt.write().await;
+                            ckpt.failed_ranges.push((range.start, range.end, format!("max retries exceeded")));
+                        }
+                    }
+                }
+            }
+        });
+        handles.push(handle);
+    }
+
+    // Wait for all pipelines
+    for handle in handles {
+        let _ = handle.await;
+    }
+
+    // Check for dead letters
+    let dead = dead_letter.read().await;
+    if !dead.is_empty() {
+        tracing::warn!("supervisor: {} ranges failed permanently", dead.len());
+    }
+
+    // Assemble vectors in order
+    let vectors_map = all_vectors.read().await;
+    let mut sorted_starts: Vec<usize> = vectors_map.keys().cloned().collect();
+    sorted_starts.sort();
+
+    let mut final_vectors: Vec<Vec<f64>> = Vec::with_capacity(total);
+    for start in sorted_starts {
+        final_vectors.extend(vectors_map[&start].clone());
+    }
+
+    let embedded_count = final_vectors.len();
+    tracing::info!("supervisor: {embedded_count}/{total} chunks embedded, storing index");
+
+    // Store — only the successfully embedded chunks
+    // We need to match chunks to vectors
+    let successful_chunks: Vec<TextChunk> = {
+        let ckpt = checkpoint.read().await;
+        let mut result = Vec::new();
+        for (s, e) in &ckpt.completed_ranges {
+            result.extend(chunks[*s..*e].iter().cloned());
+        }
+        result
+    };
+
+    if successful_chunks.is_empty() {
+        return Err("no chunks were successfully embedded".into());
+    }
+
+    let key = store::store_embeddings(obj_store, index_name, &successful_chunks, &final_vectors).await?;
+
+    // Clean up checkpoint
+    let _ = storaged::ops::delete(obj_store, &format!("checkpoints/{job_id}.json")).await;
+
+    let elapsed = start_time.elapsed().as_secs_f32();
+    let rate = embedded_count as f32 / elapsed;
+    tracing::info!("supervisor: completed — {embedded_count} vectors in {elapsed:.0}s ({rate:.0}/sec)");
+
+    Ok(key)
+}
+
+/// Embed a range of chunks. Returns vectors or error.
+async fn embed_range(
+    chunks: &[TextChunk],
+    ai_client: &AiClient,
+    batch_size: usize,
+) -> Result<Vec<Vec<f64>>, String> {
+    let mut vectors = Vec::with_capacity(chunks.len());
+
+    for batch in chunks.chunks(batch_size) {
+        let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
+        let resp = ai_client.embed(EmbedRequest {
+            texts,
+            model: None,
+        }).await.map_err(|e| format!("embed error: {e}"))?;
+        vectors.extend(resp.embeddings);
+    }
+
+    Ok(vectors)
+}
@@ -0,0 +1,72 @@
+{
+  "id": "0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1",
+  "name": "rust_in_tech_stack",
+  "schema_fingerprint": "314ee8ede68287240d7cec1403d252787c8ceb4ee2afa72cffcb0853ac3ac889",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/rust_in_tech_stack.parquet",
+      "size_bytes": 99623,
+      "created_at": "2026-03-28T03:32:50.769271381Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:32:50.769272195Z",
+  "updated_at": "2026-03-28T03:32:50.769710478Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Rust in Tech Stack.pdf",
+    "ingest_job": "ingest-1774668770769",
+    "ingest_timestamp": "2026-03-28T03:32:50.769271381Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 49
+}
@@ -0,0 +1,23 @@
+{
+  "id": "0fd78303-9ad4-45fd-90d7-db95607d9ab1",
+  "name": "timesheets",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/timesheets.parquet",
+      "size_bytes": 17539932,
+      "created_at": "2026-03-27T14:42:43.922019299Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:43.922025703Z",
+  "updated_at": "2026-03-27T14:42:43.922025703Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "1339f3d6-7677-47fb-8182-5f8e43f27cde",
+  "name": "job_orders",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/job_orders.parquet",
+      "size_bytes": 905534,
+      "created_at": "2026-03-27T14:42:38.935718195Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:38.935724058Z",
+  "updated_at": "2026-03-27T14:42:38.935724058Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "1d8a065e-59c1-45ce-967b-398bc8370cbb",
+  "name": "candidates",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/candidates.parquet",
+      "size_bytes": 10592165,
+      "created_at": "2026-03-27T14:42:38.823368759Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:38.823374843Z",
+  "updated_at": "2026-03-27T14:42:38.823374843Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -1,15 +0,0 @@
-{
-  "id": "1f81445a-404f-48ea-bd72-00f6721ee18b",
-  "name": "employees",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/employees.parquet",
-      "size_bytes": 1424,
-      "created_at": "2026-03-27T11:55:55.013996293Z"
-    }
-  ],
-  "created_at": "2026-03-27T11:55:55.014010258Z",
-  "updated_at": "2026-03-27T11:55:55.014010258Z"
-}
@@ -0,0 +1,58 @@
+{
+  "id": "24a05512-dc06-4884-9889-2ea978abdaec",
+  "name": "processed_extractions",
+  "schema_fingerprint": "c2853ba4c0cc5a897b37680fefea2708f51d51774226b00b72444e79d594c874",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/processed_extractions.parquet",
+      "size_bytes": 28175,
+      "created_at": "2026-03-28T05:16:57.437041979Z"
+    }
+  ],
+  "created_at": "2026-03-28T05:16:57.437042648Z",
+  "updated_at": "2026-03-28T05:16:57.437403737Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "type",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "processed_extractions.csv",
+    "ingest_job": "ingest-1774675017437",
+    "ingest_timestamp": "2026-03-28T05:16:57.437041979Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 543
+}
@@ -0,0 +1,65 @@
+{
+  "id": "3b12ae24-17d4-4325-92bf-f3155982f3bf",
+  "name": "users",
+  "schema_fingerprint": "30c0e31f0963e6f4af02131bbb9ea246fbbd068b849b833565a4b28211fbc90b",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/users.parquet",
+      "size_bytes": 2012,
+      "created_at": "2026-03-28T01:38:59.904968123Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:38:59.904968680Z",
+  "updated_at": "2026-03-28T01:38:59.905221340Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "id",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "username",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "password_hash",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "created_at",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "postgresql",
+    "source_file": "127.0.0.1:5432/knowledge_base.users",
+    "ingest_job": "pg-import-1774661939904",
+    "ingest_timestamp": "2026-03-28T01:38:59.904968123Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 1
+}
@@ -0,0 +1,93 @@
+{
+  "id": "3c2579b4-f3f3-4875-95fa-58d8b49ad94c",
+  "name": "meta_runs",
+  "schema_fingerprint": "68f2c0d7a3ceb0aaa3c17c64900704519c72d213161bc9e5179c42ee53f6d0df",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/meta_runs.parquet",
+      "size_bytes": 729773,
+      "created_at": "2026-03-28T01:38:57.380576453Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:38:57.380577270Z",
+  "updated_at": "2026-03-28T01:38:57.380846224Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "id",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "pipeline_id",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "iteration",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "stage_results",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "final_output",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "score",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "model_config",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "duration_ms",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "created_at",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "postgresql",
+    "source_file": "127.0.0.1:5432/knowledge_base.meta_runs",
+    "ingest_job": "pg-import-1774661937380",
+    "ingest_timestamp": "2026-03-28T01:38:57.380576453Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 38
+}
@@ -0,0 +1,58 @@
+{
+  "id": "3ead1a7a-810b-49d1-add8-752bab52dfae",
+  "name": "processed_validated",
+  "schema_fingerprint": "8078de214059a2a4582203b32c1172fcc311c2d3c762a3b075c07268308f1841",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/processed_validated.parquet",
+      "size_bytes": 15271,
+      "created_at": "2026-03-28T05:16:57.439472748Z"
+    }
+  ],
+  "created_at": "2026-03-28T05:16:57.439473552Z",
+  "updated_at": "2026-03-28T05:16:57.439749766Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "type",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "processed_validated.csv",
+    "ingest_job": "ingest-1774675017439",
+    "ingest_timestamp": "2026-03-28T05:16:57.439472748Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 60
+}
@@ -1,15 +0,0 @@
-{
-  "id": "50d402a0-6fb4-4849-ac75-3541b8530aba",
-  "name": "u5",
-  "schema_fingerprint": "from-ui",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/events.parquet",
-      "size_bytes": 0,
-      "created_at": "2026-03-27T12:16:57.520054490Z"
-    }
-  ],
-  "created_at": "2026-03-27T12:16:57.520066991Z",
-  "updated_at": "2026-03-27T12:16:57.520066991Z"
-}
@@ -0,0 +1,72 @@
+{
+  "id": "54cc1ae9-14fe-40f6-88e4-7c623f57b577",
+  "name": "crm_development_options",
+  "schema_fingerprint": "bb04328fa99161403fa0993f59bf14adcf08eb551c7b0c5e9b9e305e5e0c8556",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/crm_development_options.parquet",
+      "size_bytes": 50528,
+      "created_at": "2026-03-28T03:30:29.036504470Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:30:29.036504841Z",
+  "updated_at": "2026-03-28T03:30:29.036642992Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "CRM Development Options.pdf",
+    "ingest_job": "ingest-1774668629036",
+    "ingest_timestamp": "2026-03-28T03:30:29.036504470Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 25
+}
@@ -0,0 +1,72 @@
+{
+  "id": "5d4bbdb1-fb53-4cea-86d5-309fe067111c",
+  "name": "legacy_staffing_crm_search",
+  "schema_fingerprint": "87b3fa58cae74468bf7fbbe32397b1a88bc281832bb0226b9eae5ce2501cda4e",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/legacy_staffing_crm_search.parquet",
+      "size_bytes": 2722188,
+      "created_at": "2026-03-28T03:34:14.444384147Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:34:14.444384964Z",
+  "updated_at": "2026-03-28T03:34:14.444631183Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Legacy Staffing CRM Search.pdf",
+    "ingest_job": "ingest-1774668854444",
+    "ingest_timestamp": "2026-03-28T03:34:14.444384147Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 1
+}
@@ -0,0 +1,51 @@
+{
+  "id": "667478c3-47f9-4076-bab2-8293999d4db2",
+  "name": "knowledge_insights",
+  "schema_fingerprint": "36896987c7df0e78f5282208e3b8bbbaf9c017588d0bb42a20a7dbe97d8d4fa0",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_insights.parquet",
+      "size_bytes": 13099,
+      "created_at": "2026-03-28T04:06:05.361963023Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.361963273Z",
+  "updated_at": "2026-03-28T04:06:05.362127023Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "insight",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_insights.csv",
+    "ingest_job": "ingest-1774670765361",
+    "ingest_timestamp": "2026-03-28T04:06:05.361963023Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 100
+}
@@ -1,15 +0,0 @@
-{
-  "id": "6c8dabf2-a6b3-4e23-b7d7-dc3b37a5e1a8",
-  "name": "events",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/events.parquet",
-      "size_bytes": 1127,
-      "created_at": "2026-03-27T11:55:55.017003417Z"
-    }
-  ],
-  "created_at": "2026-03-27T11:55:55.017011224Z",
-  "updated_at": "2026-03-27T11:55:55.017011224Z"
-}
@@ -0,0 +1,51 @@
+{
+  "id": "799c825c-950e-480e-a38e-4407377e43e3",
+  "name": "knowledge_topics",
+  "schema_fingerprint": "356c37c8fef56ce7f7882c80f1301cc2f70bcbf874719cb0628b42179874494c",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_topics.parquet",
+      "size_bytes": 14252,
+      "created_at": "2026-03-28T04:06:05.358571822Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.358572461Z",
+  "updated_at": "2026-03-28T04:06:05.358833280Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "topic",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_topics.csv",
+    "ingest_job": "ingest-1774670765358",
+    "ingest_timestamp": "2026-03-28T04:06:05.358571822Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 496
+}
@@ -0,0 +1,72 @@
+{
+  "id": "7ca19535-838e-4f79-aa6f-d8ebaa2e6edc",
+  "name": "code_sorting_assistance",
+  "schema_fingerprint": "47e317d38bbad23688f6c274cbecc3dfd7076e42a6a0d09002a0bef835789bc2",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/code_sorting_assistance.parquet",
+      "size_bytes": 122034,
+      "created_at": "2026-03-28T03:31:08.585681049Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:31:08.585682007Z",
+  "updated_at": "2026-03-28T03:31:08.585978498Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Code Sorting Assistance.pdf",
+    "ingest_job": "ingest-1774668668585",
+    "ingest_timestamp": "2026-03-28T03:31:08.585681049Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 70
+}
@@ -0,0 +1,51 @@
+{
+  "id": "80eaf246-778b-4987-9bef-62a6cccb6021",
+  "name": "knowledge_questions",
+  "schema_fingerprint": "d86b4ddf7a9694118aeff3d8aa41312e0c3091b0f15cad1079ee9561c29f9c91",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_questions.parquet",
+      "size_bytes": 17253,
+      "created_at": "2026-03-28T04:06:05.363389625Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.363390263Z",
+  "updated_at": "2026-03-28T04:06:05.363439745Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "question",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_questions.csv",
+    "ingest_job": "ingest-1774670765363",
+    "ingest_timestamp": "2026-03-28T04:06:05.363389625Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 200
+}
@@ -0,0 +1,72 @@
+{
+  "id": "8f5c82bf-93ec-4ce7-b2d1-61e559242add",
+  "name": "llms3_update_prompt",
+  "schema_fingerprint": "77cfcdb5e22f095302bb816abd4e33380a6b357cd7d23efee5076083020fbf32",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/llms3_update_prompt.parquet",
+      "size_bytes": 39396,
+      "created_at": "2026-03-28T03:28:04.752204191Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:28:04.752204734Z",
+  "updated_at": "2026-03-28T03:28:04.752281926Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "LLMS3 Update Prompt.pdf",
+    "ingest_job": "ingest-1774668484752",
+    "ingest_timestamp": "2026-03-28T03:28:04.752204191Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 6
+}
@@ -0,0 +1,23 @@
+{
+  "id": "94a8bd16-6756-43af-b951-09a9e6b8300f",
+  "name": "clients",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/clients.parquet",
+      "size_bytes": 21971,
+      "created_at": "2026-03-27T14:42:38.830329102Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:38.830331694Z",
+  "updated_at": "2026-03-27T14:42:38.830331694Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "9c4d9116-1d9d-4afd-a8d1-c514a678e5fa",
+  "name": "call_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/call_log.parquet",
+      "size_bytes": 35951077,
+      "created_at": "2026-03-27T14:42:47.395548205Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:47.395555326Z",
+  "updated_at": "2026-03-27T14:42:47.395555326Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,23 @@
+{
+  "id": "d35c7941-37e2-4bde-8226-5cf69c74931a",
+  "name": "placements",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/placements.parquet",
+      "size_bytes": 1213820,
+      "created_at": "2026-03-27T14:42:39.040983450Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:39.040989351Z",
+  "updated_at": "2026-03-27T14:42:39.040989351Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,72 @@
+{
+  "id": "d609d775-1a8e-4114-8aee-3238017229a7",
+  "name": "hybrid_indexing_setup",
+  "schema_fingerprint": "dfb09f4395d131e5fda5dab65f8c9dbe395d46d0e828f504e053cdb1cf23f24b",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/hybrid_indexing_setup.parquet",
+      "size_bytes": 131530,
+      "created_at": "2026-03-28T03:30:28.978970613Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:30:28.978971805Z",
+  "updated_at": "2026-03-28T03:30:28.979244113Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Hybrid Indexing Setup.pdf",
+    "ingest_job": "ingest-1774668628978",
+    "ingest_timestamp": "2026-03-28T03:30:28.978970613Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 69
+}
@@ -0,0 +1,100 @@
+{
+  "id": "e2a8f88a-59f6-40c7-a45b-e23d8f3533b6",
+  "name": "lab_trials",
+  "schema_fingerprint": "1d5782349402439a7e44efd0ccab9ae64ac3044221adef9e828b60b8bbb44dd5",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/lab_trials.parquet",
+      "size_bytes": 64646,
+      "created_at": "2026-03-28T01:14:03.026116573Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:14:03.026117277Z",
+  "updated_at": "2026-03-28T01:14:03.026247826Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "id",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "experiment_id",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "trial_num",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "config_diff",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "config_snapshot",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "scores",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "avg_score",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "improved",
+      "data_type": "Boolean",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "duration_ms",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "created_at",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "postgresql",
+    "source_file": "127.0.0.1:5432/knowledge_base.lab_trials",
+    "ingest_job": "pg-import-1774660443026",
+    "ingest_timestamp": "2026-03-28T01:14:03.026116573Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 40
+}
@@ -0,0 +1,240 @@
+{
+  "id": "e7304f05-5278-4e17-961a-51f2588fd2aa",
+  "name": "threat_intel",
+  "schema_fingerprint": "df1e126046147b3de42086880e10c3501a3a615ecddf336bc24957a24c321241",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/threat_intel.parquet",
+      "size_bytes": 111130,
+      "created_at": "2026-03-28T01:14:03.054140697Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:14:03.054141294Z",
+  "updated_at": "2026-03-28T01:14:03.054427047Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "id",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "ip",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "threat_level",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "classification",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "confidence",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "summary",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "indicators",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "recommendation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "pattern",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "attack_type",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "likely_automated",
+      "data_type": "Boolean",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "country",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "country_code",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "isp",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "org",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "asn",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "is_proxy",
+      "data_type": "Boolean",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "is_hosting",
+      "data_type": "Boolean",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "open_ports",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "blocklist_count",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "blocklist_total",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "blocklists_blocked",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "reverse_dns",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "traceroute",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "log_count",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "banned",
+      "data_type": "Boolean",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "enriched_at",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "updated_at",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "raw_data",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "postgresql",
+    "source_file": "127.0.0.1:5432/knowledge_base.threat_intel",
+    "ingest_job": "pg-import-1774660443054",
+    "ingest_timestamp": "2026-03-28T01:14:03.054140697Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 20
+}
@@ -0,0 +1,23 @@
+{
+  "id": "e959ca11-9f6b-4843-864a-cc3f50a8aa60",
+  "name": "email_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/email_log.parquet",
+      "size_bytes": 16768671,
+      "created_at": "2026-03-27T14:42:49.271082991Z"
+    }
+  ],
+  "created_at": "2026-03-27T14:42:49.271091077Z",
+  "updated_at": "2026-03-27T14:42:49.271091077Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null
+}
@@ -0,0 +1,79 @@
+{
+  "id": "f0b6f408-71a5-4365-bd1d-98c1e176096a",
+  "name": "new_candidates",
+  "schema_fingerprint": "0fcee1ac176b9bc45ab392bbd4401042803eb80d0de48b1fa6a5e20ffb27fa8d",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/new_candidates.parquet",
+      "size_bytes": 2731,
+      "created_at": "2026-03-28T01:04:22.787759218Z"
+    }
+  ],
+  "created_at": "2026-03-28T01:04:22.787759835Z",
+  "updated_at": "2026-03-28T01:04:22.788113964Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "email",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "skills",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "years",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "new_candidates.csv",
+    "ingest_job": "ingest-1774659862787",
+    "ingest_timestamp": "2026-03-28T01:04:22.787759218Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 3
+}
@@ -1,15 +0,0 @@
-{
-  "id": "f5281277-afe9-456b-b5fa-d667c2b0f41f",
-  "name": "products",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/products.parquet",
-      "size_bytes": 1341,
-      "created_at": "2026-03-27T11:55:55.019096056Z"
-    }
-  ],
-  "created_at": "2026-03-27T11:55:55.019098965Z",
-  "updated_at": "2026-03-27T11:55:55.019098965Z"
-}
@@ -0,0 +1,72 @@
+{
+  "id": "f9055d87-c7c5-4109-b311-11fab418541f",
+  "name": "spark_vs_flink_iceberg",
+  "schema_fingerprint": "e3f74df7c9db965c184ff5f5ec92b0f68168a7f6e367e8253d28fca571a11688",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/spark_vs_flink_iceberg.parquet",
+      "size_bytes": 163762,
+      "created_at": "2026-03-28T03:35:18.667429709Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:35:18.667430594Z",
+  "updated_at": "2026-03-28T03:35:18.667593640Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Spark vs Flink Iceberg.pdf",
+    "ingest_job": "ingest-1774668918667",
+    "ingest_timestamp": "2026-03-28T03:35:18.667429709Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 124
+}
@@ -0,0 +1,72 @@
+{
+  "id": "faef511e-4f34-4002-b2f4-8aecb8cfb80e",
+  "name": "hybrid_ai_data_lake",
+  "schema_fingerprint": "1fa158168cf5f542e9cdeec30c9e9aad9b12285b66e6fbdb7321e5d9f7039515",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/hybrid_ai_data_lake.parquet",
+      "size_bytes": 220352,
+      "created_at": "2026-03-28T03:35:16.027063136Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:35:16.027064491Z",
+  "updated_at": "2026-03-28T03:35:16.027333389Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Hybrid AI Data Lake.pdf",
+    "ingest_job": "ingest-1774668916027",
+    "ingest_timestamp": "2026-03-28T03:35:16.027063136Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 133
+}
@@ -1,15 +0,0 @@
-{
-  "id": "fc046131-8f44-428a-af90-ebf6347d4cc6",
-  "name": "u",
-  "schema_fingerprint": "from-ui",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/employees.parquet",
-      "size_bytes": 0,
-      "created_at": "2026-03-27T12:16:51.462058823Z"
-    }
-  ],
-  "created_at": "2026-03-27T12:16:51.462071311Z",
-  "updated_at": "2026-03-27T12:16:51.462071311Z"
-}
@@ -0,0 +1,51 @@
+{
+  "id": "fe6b6872-03f8-491a-b2c2-7d25b19cd9bb",
+  "name": "knowledge_decisions",
+  "schema_fingerprint": "be65fb1ba2121bcf43c195c06dbdc0ae892eb835f4a909e9c1c0438ac81a1d0d",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_decisions.parquet",
+      "size_bytes": 21274,
+      "created_at": "2026-03-28T04:06:05.360578855Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.360579258Z",
+  "updated_at": "2026-03-28T04:06:05.360801969Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "decision",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_decisions.csv",
+    "ingest_job": "ingest-1774670765360",
+    "ingest_timestamp": "2026-03-28T04:06:05.360578855Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 300
+}
--- a/Show More
+++ b/Show More