Phase 28-36 body of work

Accumulated since a6f12e2 (Phase 21 Rust port + Phase 27 versioning):

- Phase 36: embed_semaphore on VectorState (permits=1) serializes
  seed embed calls — prevents sidecar socket collisions under
  concurrent /seed stress load
- Phase 31+: run_stress.ts 6-task diverse stress scaffolding;
  run_e2e_rated.ts + orchestrator.ts tightening
- Catalog dedupe cleanup: 16 duplicate manifests removed; canonical
  candidates.parquet (10.5MB -> 76KB) + placements.parquet (1.2MB ->
  11KB) regenerated post-dedupe; fresh manifests for active datasets
- vectord: harness EvalSet refinements (+181), agent portfolio
  rotation + ingest triggers (+158), autotune + rag adjustments
- catalogd/storaged/ingestd/mcp-server: misc tightening
- docs: Phase 28-36 PRD entries + DECISIONS ADR additions;
  control-plane pivot banner added to top of docs/PRD.md (pointing
  at docs/CONTROL_PLANE_PRD.md which lands in next commit)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

data/_catalog/manifests/07c119c3-2e59-463c-89a8-7980a849b6d6.json

@@ -0,0 +1,159 @@
+{
+  "id": "07c119c3-2e59-463c-89a8-7980a849b6d6",
+  "name": "workers_500k",
+  "schema_fingerprint": "71686e497f5ee2fc77d50b13c39a46d89ca3d7673e9a0879e67c2db476764b25",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/workers_500k.parquet",
+      "size_bytes": 323794818,
+      "created_at": "2026-04-17T05:57:41.972944716Z"
+    }
+  ],
+  "created_at": "2026-04-17T05:57:41.972945960Z",
+  "updated_at": "2026-04-17T05:57:41.973093967Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "worker_id",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "email",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Int64",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "zip",
+      "data_type": "Int64",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "skills",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "certifications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "archetype",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "reliability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "responsiveness",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "engagement",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "compliance",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "availability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "communications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "resume_text",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "workers_500k.csv",
+    "ingest_job": "ingest-1776405461972",
+    "ingest_timestamp": "2026-04-17T05:57:41.972944716Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 500000,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/2b8e8b01-a44e-49c5-a103-a1b55898bee1.json

@@ -1,5 +1,5 @@
 {
-  "id": "443d63f1-b0ed-4d4b-8e5a-ce59c097b97b",
+  "id": "2b8e8b01-a44e-49c5-a103-a1b55898bee1",
   "name": "kb_memory_entries",
   "schema_fingerprint": "15dbebd0abb906577e11cbb73083abeb8961b8c603a2472cafa8c46100c3fb1c",
   "objects": [
@@ -7,11 +7,11 @@
       "bucket": "primary",
       "key": "datasets/kb_memory_entries.parquet",
       "size_bytes": 8795,
-      "created_at": "2026-04-17T02:54:43.330897823Z"
+      "created_at": "2026-04-17T05:22:06.356227657Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.330898330Z",
-  "updated_at": "2026-04-17T02:54:43.330972517Z",
+  "created_at": "2026-04-17T05:22:06.356228933Z",
+  "updated_at": "2026-04-17T05:22:06.356776285Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -97,8 +97,8 @@
   "lineage": {
     "source_system": "postgresql",
     "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776394483330",
-    "ingest_timestamp": "2026-04-17T02:54:43.330897823Z",
+    "ingest_job": "pg-stream-1776403326356",
+    "ingest_timestamp": "2026-04-17T05:22:06.356227657Z",
     "parent_datasets": []
   },
   "freshness": null,

data/_catalog/manifests/2db2d92f-1171-4f2f-a707-ad387b6cd464.json

@@ -0,0 +1,117 @@
+{
+  "id": "2db2d92f-1171-4f2f-a707-ad387b6cd464",
+  "name": "onboard_live_1776729919",
+  "schema_fingerprint": "9cdf2fc924bad875e63bd12a82dfb1cd080c5cbba71f671b4a2d41e746df187d",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/onboard_live_1776729919.parquet",
+      "size_bytes": 31444,
+      "created_at": "2026-04-21T00:05:19.717239967Z"
+    }
+  ],
+  "created_at": "2026-04-21T00:05:19.717241621Z",
+  "updated_at": "2026-04-21T00:05:19.717370050Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "worker_id",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "email",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "skills",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "certifications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "availability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "reliability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "archetype",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "sample_1776729919.csv",
+    "ingest_job": "ingest-1776729919717",
+    "ingest_timestamp": "2026-04-21T00:05:19.717239967Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 173,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/3178f0da-5985-4731-b343-162dd415f3be.json

@@ -1,5 +1,5 @@
 {
-  "id": "ab2b610a-cee7-40e1-9dab-c709e2292709",
+  "id": "3178f0da-5985-4731-b343-162dd415f3be",
   "name": "kb_self_reports",
   "schema_fingerprint": "76382b028b92bb0b361306dced3d773af0ee8de6aa160cedfc4d649f2860167a",
   "objects": [
@@ -7,11 +7,11 @@
       "bucket": "primary",
       "key": "datasets/kb_self_reports.parquet",
       "size_bytes": 51134,
-      "created_at": "2026-04-17T02:54:43.335457986Z"
+      "created_at": "2026-04-17T04:14:34.355419647Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.335458749Z",
-  "updated_at": "2026-04-17T02:54:43.335536869Z",
+  "created_at": "2026-04-17T04:14:34.355420701Z",
+  "updated_at": "2026-04-17T06:06:21.358604060Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -62,8 +62,8 @@
   "lineage": {
     "source_system": "postgresql",
     "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776394483335",
-    "ingest_timestamp": "2026-04-17T02:54:43.335457986Z",
+    "ingest_job": "pg-stream-1776399274355",
+    "ingest_timestamp": "2026-04-17T04:14:34.355419647Z",
     "parent_datasets": []
   },
   "freshness": null,

data/_catalog/manifests/32ee74a0-59b4-4e5b-8edb-70c9347a4bf3.json

@@ -0,0 +1,26 @@
+{
+  "id": "32ee74a0-59b4-4e5b-8edb-70c9347a4bf3",
+  "name": "successful_playbooks_live",
+  "schema_fingerprint": "4e28e94650d9f5689e952674aebfd09db70edfcba617713081c0a83bf59121ce",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/successful_playbooks_live.parquet",
+      "size_bytes": 284341,
+      "created_at": "2026-04-22T03:28:28.343840136Z"
+    }
+  ],
+  "created_at": "2026-04-20T11:07:57.308050648Z",
+  "updated_at": "2026-04-22T03:28:28.343843823Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [],
+  "lineage": null,
+  "freshness": null,
+  "tags": [],
+  "row_count": null,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/33544879-fcfa-4458-b782-a4ca8dafed08.json

@@ -0,0 +1,159 @@
+{
+  "id": "33544879-fcfa-4458-b782-a4ca8dafed08",
+  "name": "workers_100k",
+  "schema_fingerprint": "1cc2a0caf174aa9bf67b1e2cb7858a312a0c94d604d906f807eb5610864ff70e",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/workers_100k.parquet",
+      "size_bytes": 65225188,
+      "created_at": "2026-04-17T04:54:03.306737596Z"
+    }
+  ],
+  "created_at": "2026-04-17T04:54:03.306740279Z",
+  "updated_at": "2026-04-17T04:54:03.306927852Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "worker_id",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "email",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Int64",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "zip",
+      "data_type": "Int64",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "skills",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "certifications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "archetype",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "reliability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "responsiveness",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "engagement",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "compliance",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "availability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "communications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "resume_text",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "workers_100k.csv",
+    "ingest_job": "ingest-1776401643306",
+    "ingest_timestamp": "2026-04-17T04:54:03.306737596Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 100000,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/35f36c38-5cd3-438e-b93d-7ff7720f0346.json

@@ -1,5 +1,5 @@
 {
-  "id": "ce3fe55b-61cd-4ac4-95de-b91f9186d6e3",
+  "id": "35f36c38-5cd3-438e-b93d-7ff7720f0346",
   "name": "kb_meta_runs",
   "schema_fingerprint": "68f2c0d7a3ceb0aaa3c17c64900704519c72d213161bc9e5179c42ee53f6d0df",
   "objects": [
@@ -7,11 +7,11 @@
       "bucket": "primary",
       "key": "datasets/kb_meta_runs.parquet",
       "size_bytes": 886387,
-      "created_at": "2026-04-17T02:54:43.299467359Z"
+      "created_at": "2026-04-17T05:22:06.314413387Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.299468094Z",
-  "updated_at": "2026-04-17T04:23:16.134134852Z",
+  "created_at": "2026-04-17T05:22:06.314414515Z",
+  "updated_at": "2026-04-17T06:06:21.300610797Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -80,13 +80,7 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776399796133",
-    "ingest_timestamp": "2026-04-17T04:23:16.133868872Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
   "row_count": 48,

data/_catalog/manifests/36b0f141-93a1-4776-beef-b0b8d17a12a9.json

@@ -1,96 +0,0 @@
-{
-  "id": "36b0f141-93a1-4776-beef-b0b8d17a12a9",
-  "name": "demo_customers",
-  "schema_fingerprint": "7af9708f2366f790e0ae03db6f681f5a4d4b8af4d1527b091bedfee1be81d304",
-  "objects": [
-    {
-      "bucket": "primary",
-      "key": "datasets/demo_customers.parquet",
-      "size_bytes": 4583,
-      "created_at": "2026-04-17T01:35:01.636480725Z"
-    }
-  ],
-  "created_at": "2026-04-17T01:35:01.636481698Z",
-  "updated_at": "2026-04-17T01:35:01.636776595Z",
-  "description": "",
-  "owner": "",
-  "sensitivity": "pii",
-  "columns": [
-    {
-      "name": "id",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "name",
-      "data_type": "Utf8",
-      "sensitivity": "pii",
-      "description": "",
-      "is_pii": true
-    },
-    {
-      "name": "email",
-      "data_type": "Utf8",
-      "sensitivity": "pii",
-      "description": "",
-      "is_pii": true
-    },
-    {
-      "name": "city",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "tier",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "lifetime_spend",
-      "data_type": "Float64",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "is_active",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "signed_up_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "notes",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    }
-  ],
-  "lineage": {
-    "source_system": "mysql",
-    "source_file": "dsn: mysql://lh:***@127.0.0.1:3306/lh_demo",
-    "ingest_job": "scheduled-mysql-1776389701636",
-    "ingest_timestamp": "2026-04-17T01:35:01.636480725Z",
-    "parent_datasets": []
-  },
-  "freshness": null,
-  "tags": [],
-  "row_count": 11,
-  "last_embedded_at": null,
-  "embedding_stale_since": null,
-  "embedding_refresh_policy": null
-}

data/_catalog/manifests/564b00ae-cbf3-4efd-aa55-84cdb6d2b0b7.json

@@ -0,0 +1,117 @@
+{
+  "id": "564b00ae-cbf3-4efd-aa55-84cdb6d2b0b7",
+  "name": "client_workerskjkk",
+  "schema_fingerprint": "cdfe85348885ddf329e5e6e9bf0e2c75c92d1a86fdb0fd3875ed46e3f93c4d82",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/client_workerskjkk.parquet",
+      "size_bytes": 32201,
+      "created_at": "2026-04-21T00:49:04.623625149Z"
+    }
+  ],
+  "created_at": "2026-04-21T00:49:04.623626738Z",
+  "updated_at": "2026-04-21T00:49:04.623901788Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "worker_id",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "email",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "skills",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "certifications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "availability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "reliability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "archetype",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "staffing_roster_sample.csv",
+    "ingest_job": "ingest-1776732544623",
+    "ingest_timestamp": "2026-04-21T00:49:04.623625149Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 180,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/5bab2fac-8bdd-41b7-a993-ed48fecc60b3.json

@@ -0,0 +1,117 @@
+{
+  "id": "5bab2fac-8bdd-41b7-a993-ed48fecc60b3",
+  "name": "onboard_demo2",
+  "schema_fingerprint": "274575423f7d2648cea49a628a2e827dbd8822cef1f4eebfcf8e0a5f8602fbf7",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/onboard_demo2.parquet",
+      "size_bytes": 7833,
+      "created_at": "2026-04-20T23:13:36.886222056Z"
+    }
+  ],
+  "created_at": "2026-04-20T23:13:36.886223472Z",
+  "updated_at": "2026-04-20T23:13:36.886426834Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "worker_id",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "email",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "skills",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "certifications",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "availability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "reliability",
+      "data_type": "Float64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "archetype",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "staffing_roster_sample.csv",
+    "ingest_job": "ingest-1776726816886",
+    "ingest_timestamp": "2026-04-20T23:13:36.886222056Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 25,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/8192d934-fc90-46dd-b8bd-c443a4743b19.json

@@ -1,96 +0,0 @@
-{
-  "id": "8192d934-fc90-46dd-b8bd-c443a4743b19",
-  "name": "demo_customers",
-  "schema_fingerprint": "7af9708f2366f790e0ae03db6f681f5a4d4b8af4d1527b091bedfee1be81d304",
-  "objects": [
-    {
-      "bucket": "primary",
-      "key": "datasets/demo_customers.parquet",
-      "size_bytes": 4519,
-      "created_at": "2026-04-17T01:00:48.361437263Z"
-    }
-  ],
-  "created_at": "2026-04-17T01:00:48.361437860Z",
-  "updated_at": "2026-04-17T01:00:48.361681724Z",
-  "description": "",
-  "owner": "",
-  "sensitivity": "pii",
-  "columns": [
-    {
-      "name": "id",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "name",
-      "data_type": "Utf8",
-      "sensitivity": "pii",
-      "description": "",
-      "is_pii": true
-    },
-    {
-      "name": "email",
-      "data_type": "Utf8",
-      "sensitivity": "pii",
-      "description": "",
-      "is_pii": true
-    },
-    {
-      "name": "city",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "tier",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "lifetime_spend",
-      "data_type": "Float64",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "is_active",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "signed_up_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "notes",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    }
-  ],
-  "lineage": {
-    "source_system": "mysql",
-    "source_file": "dsn: mysql://lh:***@127.0.0.1:3306/lh_demo",
-    "ingest_job": "mysql-stream-1776387648361",
-    "ingest_timestamp": "2026-04-17T01:00:48.361437263Z",
-    "parent_datasets": []
-  },
-  "freshness": null,
-  "tags": [],
-  "row_count": 10,
-  "last_embedded_at": null,
-  "embedding_stale_since": null,
-  "embedding_refresh_policy": null
-}

data/_catalog/manifests/8621894c-f9c6-4eea-8e71-e80b9cdc9a36.json

@@ -1,90 +0,0 @@
-{
-  "id": "8621894c-f9c6-4eea-8e71-e80b9cdc9a36",
-  "name": "demo_customers",
-  "schema_fingerprint": "7af9708f2366f790e0ae03db6f681f5a4d4b8af4d1527b091bedfee1be81d304",
-  "objects": [
-    {
-      "bucket": "primary",
-      "key": "datasets/demo_customers.parquet",
-      "size_bytes": 4583,
-      "created_at": "2026-04-17T01:34:30.158848713Z"
-    }
-  ],
-  "created_at": "2026-04-17T01:34:30.158849587Z",
-  "updated_at": "2026-04-17T05:56:27.642214035Z",
-  "description": "",
-  "owner": "",
-  "sensitivity": null,
-  "columns": [
-    {
-      "name": "id",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "name",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "email",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "city",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "tier",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "lifetime_spend",
-      "data_type": "Float64",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "is_active",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "signed_up_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "notes",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    }
-  ],
-  "lineage": null,
-  "freshness": null,
-  "tags": [],
-  "row_count": 11,
-  "last_embedded_at": null,
-  "embedding_stale_since": null,
-  "embedding_refresh_policy": null
-}

data/_catalog/manifests/8f37454b-27c8-4f61-aeca-8e48070db552.json

@@ -1,96 +0,0 @@
-{
-  "id": "8f37454b-27c8-4f61-aeca-8e48070db552",
-  "name": "demo_customers",
-  "schema_fingerprint": "7af9708f2366f790e0ae03db6f681f5a4d4b8af4d1527b091bedfee1be81d304",
-  "objects": [
-    {
-      "bucket": "primary",
-      "key": "datasets/demo_customers.parquet",
-      "size_bytes": 4519,
-      "created_at": "2026-04-17T01:34:00.160615345Z"
-    }
-  ],
-  "created_at": "2026-04-17T01:34:00.160616631Z",
-  "updated_at": "2026-04-17T02:45:58.283076939Z",
-  "description": "",
-  "owner": "",
-  "sensitivity": "pii",
-  "columns": [
-    {
-      "name": "id",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "name",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "email",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "city",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "tier",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "lifetime_spend",
-      "data_type": "Float64",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "is_active",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "signed_up_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "notes",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    }
-  ],
-  "lineage": {
-    "source_system": "mysql",
-    "source_file": "dsn: mysql://lh:***@127.0.0.1:3306/lh_demo",
-    "ingest_job": "scheduled-mysql-1776389700165",
-    "ingest_timestamp": "2026-04-17T01:35:00.165218923Z",
-    "parent_datasets": []
-  },
-  "freshness": null,
-  "tags": [],
-  "row_count": 11,
-  "last_embedded_at": null,
-  "embedding_stale_since": null,
-  "embedding_refresh_policy": null
-}

data/_catalog/manifests/909a1fd1-0924-4a5e-a527-d249a07540a1.json

@@ -1,17 +1,17 @@
 {
-  "id": "14698884-071c-4adb-ae50-cfb8d885656c",
+  "id": "909a1fd1-0924-4a5e-a527-d249a07540a1",
   "name": "kb_response_cache",
   "schema_fingerprint": "c90d7be310b5025d2c4d398cf07692d8d9bb46ed591c0a87b339bafcac9ddeed",
   "objects": [
     {
       "bucket": "primary",
       "key": "datasets/kb_response_cache.parquet",
-      "size_bytes": 8360233,
-      "created_at": "2026-04-17T02:54:43.145342968Z"
+      "size_bytes": 8464841,
+      "created_at": "2026-04-17T05:22:06.168071783Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.145343876Z",
-  "updated_at": "2026-04-17T02:54:43.145629696Z",
+  "created_at": "2026-04-17T05:22:06.168072339Z",
+  "updated_at": "2026-04-17T06:06:21.356504054Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -94,16 +94,10 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776394483145",
-    "ingest_timestamp": "2026-04-17T02:54:43.145342968Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
-  "row_count": 195,
+  "row_count": 197,
   "last_embedded_at": null,
   "embedding_stale_since": null,
   "embedding_refresh_policy": null

data/_catalog/manifests/9861b817-f571-496b-9d7b-c4d385e51ca5.json

@@ -0,0 +1,75 @@
+{
+  "id": "9861b817-f571-496b-9d7b-c4d385e51ca5",
+  "name": "sparse_workers",
+  "schema_fingerprint": "1055d5e17fa3285a1cac7d4c949ae03699cd046af671f22d0d1cf4d0024c66a4",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/sparse_workers.parquet",
+      "size_bytes": 7619,
+      "created_at": "2026-04-17T20:31:24.968688194Z"
+    }
+  ],
+  "created_at": "2026-04-17T20:31:24.968689336Z",
+  "updated_at": "2026-04-17T20:31:24.968969734Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": "pii",
+  "columns": [
+    {
+      "name": "name",
+      "data_type": "Utf8",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "phone",
+      "data_type": "Int64",
+      "sensitivity": "pii",
+      "description": "",
+      "is_pii": true
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "city",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "state",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "notes",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "sparse_workers.csv",
+    "ingest_job": "ingest-1776457884968",
+    "ingest_timestamp": "2026-04-17T20:31:24.968688194Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 200,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/a2206c79-28d6-40f2-b97b-61a5df5bee06.json

@@ -1,17 +1,17 @@
 {
-  "id": "b076cff9-c522-48fc-b892-b7cba7b29c9b",
+  "id": "a2206c79-28d6-40f2-b97b-61a5df5bee06",
   "name": "kb_threat_intel",
   "schema_fingerprint": "df1e126046147b3de42086880e10c3501a3a615ecddf336bc24957a24c321241",
   "objects": [
     {
       "bucket": "primary",
       "key": "datasets/kb_threat_intel.parquet",
-      "size_bytes": 247112,
-      "created_at": "2026-04-17T02:54:43.321496407Z"
+      "size_bytes": 247318,
+      "created_at": "2026-04-17T04:23:16.149697926Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.321497035Z",
-  "updated_at": "2026-04-17T02:54:43.321758360Z",
+  "created_at": "2026-04-17T04:23:16.149698245Z",
+  "updated_at": "2026-04-17T05:56:44.994303201Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -227,13 +227,7 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776394483321",
-    "ingest_timestamp": "2026-04-17T02:54:43.321496407Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
   "row_count": 54,

data/_catalog/manifests/a99da7e2-8f42-4ff3-8fa7-679d4e514581.json

@@ -1,17 +1,17 @@
 {
-  "id": "eb370ff1-c037-476f-8c3f-61b96b3a6046",
+  "id": "a99da7e2-8f42-4ff3-8fa7-679d4e514581",
   "name": "kb_response_cache_history",
   "schema_fingerprint": "9a05c209a51f9543bd7dc9387695b0c67a6abc135b53eaeec458140712bfcf50",
   "objects": [
     {
       "bucket": "primary",
       "key": "datasets/kb_response_cache_history.parquet",
-      "size_bytes": 8292303,
-      "created_at": "2026-04-17T02:54:43.242879858Z"
+      "size_bytes": 8396568,
+      "created_at": "2026-04-17T04:14:34.260241351Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.242880413Z",
-  "updated_at": "2026-04-17T05:56:45.008387335Z",
+  "created_at": "2026-04-17T04:14:34.260242474Z",
+  "updated_at": "2026-04-17T06:06:21.334716946Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -69,8 +69,8 @@
   "lineage": {
     "source_system": "postgresql",
     "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776394483242",
-    "ingest_timestamp": "2026-04-17T02:54:43.242879858Z",
+    "ingest_job": "pg-stream-1776403326260",
+    "ingest_timestamp": "2026-04-17T05:22:06.260249210Z",
     "parent_datasets": []
   },
   "freshness": null,

data/_catalog/manifests/c2f0acdb-df65-4c21-9633-08af4248c85b.json

@@ -1,5 +1,5 @@
 {
-  "id": "a5858f94-267f-4382-ba32-0934e1b984f7",
+  "id": "c2f0acdb-df65-4c21-9633-08af4248c85b",
   "name": "kb_lab_trials",
   "schema_fingerprint": "1d5782349402439a7e44efd0ccab9ae64ac3044221adef9e828b60b8bbb44dd5",
   "objects": [
@@ -7,11 +7,11 @@
       "bucket": "primary",
       "key": "datasets/kb_lab_trials.parquet",
       "size_bytes": 68221,
-      "created_at": "2026-04-17T02:54:43.423304357Z"
+      "created_at": "2026-04-17T05:22:06.382106104Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.423304931Z",
-  "updated_at": "2026-04-17T04:23:16.166692762Z",
+  "created_at": "2026-04-17T05:22:06.382107354Z",
+  "updated_at": "2026-04-17T06:06:21.358304760Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -87,13 +87,7 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776399796166",
-    "ingest_timestamp": "2026-04-17T04:23:16.166457198Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
   "row_count": 41,

data/_catalog/manifests/d668a036-7b50-4845-8194-6a03ef370c8d.json

@@ -1,5 +1,5 @@
 {
-  "id": "03c62234-f9f8-40e9-a27e-d5e09ab2713d",
+  "id": "d668a036-7b50-4845-8194-6a03ef370c8d",
   "name": "kb_pipeline_runs",
   "schema_fingerprint": "c019b81feb58ff2aefe4cbe700056a100e25716f2aa8e8415ba8f20656812f75",
   "objects": [
@@ -7,11 +7,11 @@
       "bucket": "primary",
       "key": "datasets/kb_pipeline_runs.parquet",
       "size_bytes": 1518415,
-      "created_at": "2026-04-17T02:54:43.281447563Z"
+      "created_at": "2026-04-17T05:22:06.289208160Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.281448143Z",
-  "updated_at": "2026-04-17T05:56:44.999492793Z",
+  "created_at": "2026-04-17T05:22:06.289209657Z",
+  "updated_at": "2026-04-17T06:06:21.311384739Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -87,13 +87,7 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776403326289",
-    "ingest_timestamp": "2026-04-17T05:22:06.289208160Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
   "row_count": 195,

data/_catalog/manifests/d92c5d57-14f9-4ce0-9c4f-19c6db94a170.json

@@ -1,17 +1,17 @@
 {
-  "id": "7ad27f97-622e-49e3-9c38-327cb2334fa1",
+  "id": "d92c5d57-14f9-4ce0-9c4f-19c6db94a170",
   "name": "kb_team_runs",
   "schema_fingerprint": "d704ee2b9b434774aed2258da2fddcdcbab226547a011ba24d4281253657bdd3",
   "objects": [
     {
       "bucket": "primary",
       "key": "datasets/kb_team_runs.parquet",
-      "size_bytes": 19185025,
-      "created_at": "2026-04-17T02:54:43.043742588Z"
+      "size_bytes": 19293718,
+      "created_at": "2026-04-17T05:22:06.071898629Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.043743165Z",
-  "updated_at": "2026-04-17T02:54:43.043968762Z",
+  "created_at": "2026-04-17T05:22:06.071899235Z",
+  "updated_at": "2026-04-17T05:56:44.992569105Z",
   "description": "",
   "owner": "",
   "sensitivity": null,
@@ -108,16 +108,10 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776394483043",
-    "ingest_timestamp": "2026-04-17T02:54:43.043742588Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
-  "row_count": 588,
+  "row_count": 590,
   "last_embedded_at": null,
   "embedding_stale_since": null,
   "embedding_refresh_policy": null

data/_catalog/manifests/dab56959-d79b-4c68-a6bd-e5fc487621ff.json

@@ -0,0 +1,68 @@
+{
+  "id": "dab56959-d79b-4c68-a6bd-e5fc487621ff",
+  "name": "successful_playbooks",
+  "schema_fingerprint": "30b52caca7b6303852cd648882eae9ed87205b80e5e570f6fda08dc8735d9b06",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/successful_playbooks.parquet",
+      "size_bytes": 2540,
+      "created_at": "2026-04-17T22:25:34.467634304Z"
+    }
+  ],
+  "created_at": "2026-04-17T22:25:34.467635066Z",
+  "updated_at": "2026-04-19T08:44:08.533058251Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "timestamp",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "operation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "approach",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "result",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "context",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "playbook.csv",
+    "ingest_job": "ingest-1776588248532",
+    "ingest_timestamp": "2026-04-19T08:44:08.532686879Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 29,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/e7304f05-5278-4e17-961a-51f2588fd2aa.json

@@ -1,243 +0,0 @@
-{
-  "id": "e7304f05-5278-4e17-961a-51f2588fd2aa",
-  "name": "threat_intel",
-  "schema_fingerprint": "df1e126046147b3de42086880e10c3501a3a615ecddf336bc24957a24c321241",
-  "objects": [
-    {
-      "bucket": "primary",
-      "key": "datasets/threat_intel.parquet",
-      "size_bytes": 111130,
-      "created_at": "2026-03-28T01:14:03.054140697Z"
-    }
-  ],
-  "created_at": "2026-03-28T01:14:03.054141294Z",
-  "updated_at": "2026-04-17T05:56:45.009810081Z",
-  "description": "",
-  "owner": "",
-  "sensitivity": null,
-  "columns": [
-    {
-      "name": "id",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "ip",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "threat_level",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "classification",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "confidence",
-      "data_type": "Float64",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "summary",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "indicators",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "recommendation",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "pattern",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "attack_type",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "likely_automated",
-      "data_type": "Boolean",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "country",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "country_code",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "city",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "isp",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "org",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "asn",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "is_proxy",
-      "data_type": "Boolean",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "is_hosting",
-      "data_type": "Boolean",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "open_ports",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "blocklist_count",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "blocklist_total",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "blocklists_blocked",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "reverse_dns",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "traceroute",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "log_count",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "banned",
-      "data_type": "Boolean",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "enriched_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "updated_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "raw_data",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    }
-  ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://postgres@127.0.0.1:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776326353882",
-    "ingest_timestamp": "2026-04-16T07:59:13.882669337Z",
-    "parent_datasets": []
-  },
-  "freshness": null,
-  "tags": [],
-  "row_count": 20,
-  "last_embedded_at": "2026-04-16T15:08:32.348412159Z",
-  "embedding_stale_since": null,
-  "embedding_refresh_policy": null
-}

data/_catalog/manifests/eeb0d288-e227-478d-9407-78a43b623c5c.json

@@ -1,5 +1,5 @@
 {
-  "id": "765985e6-ea16-46b5-a6c0-ddaff346827f",
+  "id": "eeb0d288-e227-478d-9407-78a43b623c5c",
   "name": "kb_meta_pipelines",
   "schema_fingerprint": "cabe1d7fc442e1dfcfaabf663509d590c6edc84b445a91acaf0ae68b94aff518",
   "objects": [
@@ -7,14 +7,14 @@
       "bucket": "primary",
       "key": "datasets/kb_meta_pipelines.parquet",
       "size_bytes": 32085,
-      "created_at": "2026-04-17T02:54:43.307966396Z"
+      "created_at": "2026-04-17T04:14:34.315211960Z"
     }
   ],
-  "created_at": "2026-04-17T02:54:43.307966799Z",
-  "updated_at": "2026-04-17T04:23:16.140272666Z",
+  "created_at": "2026-04-17T04:14:34.315212764Z",
+  "updated_at": "2026-04-17T06:06:21.357779083Z",
   "description": "",
   "owner": "",
-  "sensitivity": "pii",
+  "sensitivity": null,
   "columns": [
     {
       "name": "id",
@@ -26,9 +26,9 @@
     {
       "name": "name",
       "data_type": "Utf8",
-      "sensitivity": "pii",
+      "sensitivity": null,
       "description": "",
-      "is_pii": true
+      "is_pii": false
     },
     {
       "name": "data_source",
@@ -108,13 +108,7 @@
       "is_pii": false
     }
   ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776399796139",
-    "ingest_timestamp": "2026-04-17T04:23:16.139992290Z",
-    "parent_datasets": []
-  },
+  "lineage": null,
   "freshness": null,
   "tags": [],
   "row_count": 6,

data/_catalog/manifests/ef6c0acf-de7a-4be0-877e-e0c2889f390f.json

@@ -7,11 +7,11 @@
       "bucket": "primary",
       "key": "datasets/threat_intel.parquet",
       "size_bytes": 247112,
-      "created_at": "2026-04-16T07:59:13.882669337Z"
+      "created_at": "2026-04-19T11:19:40.517865325Z"
     }
   ],
   "created_at": "2026-04-16T07:59:13.882669807Z",
-  "updated_at": "2026-04-17T06:06:21.359639461Z",
+  "updated_at": "2026-04-19T11:19:40.517873046Z",
   "description": "",
   "owner": "",
   "sensitivity": null,

data/_catalog/manifests/f2fda821-2f86-4c81-af58-488d184d8243.json

@@ -0,0 +1,47 @@
+{
+  "id": "f2fda821-2f86-4c81-af58-488d184d8243",
+  "name": "resumes",
+  "schema_fingerprint": "a2b2c191412faf317800637da5887d30d749d5ba0222d0a8edd12cde08daeb8a",
+  "objects": [
+    {
+      "bucket": "primary",
+      "key": "datasets/resumes.parquet",
+      "size_bytes": 3159,
+      "created_at": "2026-04-19T11:26:29.607950059Z"
+    }
+  ],
+  "created_at": "2026-04-19T11:26:29.607951447Z",
+  "updated_at": "2026-04-19T11:26:29.608054435Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "doc_id",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "resume_text",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "ndjson",
+    "source_file": "resumes.ndjson",
+    "ingest_job": "ingest-1776597989607",
+    "ingest_timestamp": "2026-04-19T11:26:29.607950059Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 10,
+  "last_embedded_at": null,
+  "embedding_stale_since": null,
+  "embedding_refresh_policy": null
+}

data/_catalog/manifests/f429ac10-bc7d-41cf-b30f-a9590760ee32.json

@@ -1,124 +0,0 @@
-{
-  "id": "f429ac10-bc7d-41cf-b30f-a9590760ee32",
-  "name": "kb_team_runs",
-  "schema_fingerprint": "d704ee2b9b434774aed2258da2fddcdcbab226547a011ba24d4281253657bdd3",
-  "objects": [
-    {
-      "bucket": "primary",
-      "key": "datasets/kb_team_runs.parquet",
-      "size_bytes": 19616539,
-      "created_at": "2026-04-16T06:07:08.188675427Z"
-    }
-  ],
-  "created_at": "2026-04-16T06:07:08.188676317Z",
-  "updated_at": "2026-04-17T05:56:27.639888971Z",
-  "description": "",
-  "owner": "",
-  "sensitivity": null,
-  "columns": [
-    {
-      "name": "id",
-      "data_type": "Int32",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "mode",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "prompt",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "config",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "responses",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "models_used",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "created_at",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "archived",
-      "data_type": "Boolean",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "tags",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "notes",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "quality_score",
-      "data_type": "Float64",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "score_method",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    },
-    {
-      "name": "score_metadata",
-      "data_type": "Utf8",
-      "sensitivity": null,
-      "description": "",
-      "is_pii": false
-    }
-  ],
-  "lineage": {
-    "source_system": "postgresql",
-    "source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
-    "ingest_job": "pg-stream-1776403326071",
-    "ingest_timestamp": "2026-04-17T05:22:06.071898629Z",
-    "parent_datasets": []
-  },
-  "freshness": null,
-  "tags": [],
-  "row_count": 590,
-  "last_embedded_at": "2026-04-16T15:08:32.471504656Z",
-  "embedding_stale_since": "2026-04-17T05:22:06.072418149Z",
-  "embedding_refresh_policy": null
-}