Observer KB: 477 pages from 8 ChatGPT PDFs indexed and searchable

- Fixed Unicode crash in chunker (ligatures like fi causing byte boundary panic) - Chunker now works with char indices, not byte indices - 8 PDFs ingested: hybrid_ai_data_lake (133p), spark_vs_flink_iceberg (124p), code_sorting (70p), hybrid_indexing (69p), rust_in_tech_stack (49p), etc. - 477 chunks embedded via supervisor (43/sec) - HNSW index built in 0.44s - Search results <200ms, finding correct conversations by meaning - Tested: "S3 architecture", "CRM WordPress", "Rust vs Python" — all correct Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 22:53:27 -05:00
parent c49b30053b
commit 8ca4ba5717
25 changed files with 657 additions and 24 deletions
@@ -0,0 +1,72 @@
+{
+  "id": "7ca19535-838e-4f79-aa6f-d8ebaa2e6edc",
+  "name": "code_sorting_assistance",
+  "schema_fingerprint": "47e317d38bbad23688f6c274cbecc3dfd7076e42a6a0d09002a0bef835789bc2",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/code_sorting_assistance.parquet",
+      "size_bytes": 122034,
+      "created_at": "2026-03-28T03:31:08.585681049Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:31:08.585682007Z",
+  "updated_at": "2026-03-28T03:31:08.585978498Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Code Sorting Assistance.pdf",
+    "ingest_job": "ingest-1774668668585",
+    "ingest_timestamp": "2026-03-28T03:31:08.585681049Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 70
+}