Phase 7: Vector index + RAG pipeline

- vectord crate: chunk → embed → store → search → RAG - chunker: configurable chunk size + overlap, sentence-boundary aware splitting - store: embeddings as Parquet (binary blob f32 vectors), portable format - search: brute-force cosine similarity (works up to ~100K vectors) - rag: full pipeline — embed question → search index → retrieve context → LLM answer - Endpoints: POST /vectors/index, /vectors/search, /vectors/rag - Gateway wired with vectord service - Tested: 200 candidate resumes indexed in 5.4s, semantic search + RAG working - 20 unit tests passing (chunker, search, ingestd, shared) - AI gives honest "no match found" when context doesn't support an answer Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 08:12:28 -05:00
parent bb05c4412e
commit 26fc98c885
30 changed files with 771 additions and 31 deletions
@@ -0,0 +1,15 @@
+{
+  "id": "0927b27a-80a9-4790-a34f-bda7ff176aac",
+  "name": "job_orders",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/job_orders.parquet",
+      "size_bytes": 225889,
+      "created_at": "2026-03-27T13:11:41.384341257Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:41.384344032Z",
+  "updated_at": "2026-03-27T13:11:41.384344032Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "0bf1eb1f-b182-4025-9b44-b8553e678bcf",
+  "name": "timesheets",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/timesheets.parquet",
+      "size_bytes": 2458229,
+      "created_at": "2026-03-27T13:11:42.084209718Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:42.084217486Z",
+  "updated_at": "2026-03-27T13:11:42.084217486Z"
+}
@@ -1,15 +0,0 @@
-{
-  "id": "1ca61945-d151-490b-81fd-2ca0397b68fa",
-  "name": "sms_messages",
-  "schema_fingerprint": "e1d079cbb2b7eedae5019767a886bd9a3396e291aa03630b9db69e9864948c09",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/sms_messages.parquet",
-      "size_bytes": 2018,
-      "created_at": "2026-03-27T13:07:14.253881797Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:07:14.253886027Z",
-  "updated_at": "2026-03-27T13:07:14.253886027Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "478072c3-0c95-46a2-9193-f4b3ac4085ab",
-  "name": "test_ingest",
-  "schema_fingerprint": "4bdc4e5baeddc1187aecd4bfb788654f26145c2ba346b4bec6ca8ab950e1c133",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/test_ingest.parquet",
-      "size_bytes": 3129,
-      "created_at": "2026-03-27T13:06:57.437484309Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:06:57.437488259Z",
-  "updated_at": "2026-03-27T13:06:57.437488259Z"
-}
@@ -0,0 +1,15 @@
+{
+  "id": "4be87c74-10b4-463c-b69d-f20c9cd18ed7",
+  "name": "candidates",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/candidates.parquet",
+      "size_bytes": 2003395,
+      "created_at": "2026-03-27T13:11:41.341589905Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:41.341599187Z",
+  "updated_at": "2026-03-27T13:11:41.341599187Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "75bb6855-488b-4300-89c2-970871bd99cc",
+  "name": "email_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/email_log.parquet",
+      "size_bytes": 1873775,
+      "created_at": "2026-03-27T13:11:42.757205427Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:42.757211105Z",
+  "updated_at": "2026-03-27T13:11:42.757211105Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "ad393eee-ba0c-4338-9a8b-236bba3816ac",
+  "name": "placements",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/placements.parquet",
+      "size_bytes": 217395,
+      "created_at": "2026-03-27T13:11:41.433628136Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:41.433633927Z",
+  "updated_at": "2026-03-27T13:11:41.433633927Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "b334b1eb-d7a2-473f-a7fa-017b17de74bd",
+  "name": "clients",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/clients.parquet",
+      "size_bytes": 34228,
+      "created_at": "2026-03-27T13:11:41.350247882Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:41.350250705Z",
+  "updated_at": "2026-03-27T13:11:41.350250705Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "e015f0e2-51e4-4301-855d-76c54992c5b9",
+  "name": "call_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/call_log.parquet",
+      "size_bytes": 3276693,
+      "created_at": "2026-03-27T13:11:42.483220340Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:11:42.483225870Z",
+  "updated_at": "2026-03-27T13:11:42.483225870Z"
+}