Observer KB: 477 pages from 8 ChatGPT PDFs indexed and searchable

- Fixed Unicode crash in chunker (ligatures like fi causing byte boundary panic)
- Chunker now works with char indices, not byte indices
- 8 PDFs ingested: hybrid_ai_data_lake (133p), spark_vs_flink_iceberg (124p),
  code_sorting (70p), hybrid_indexing (69p), rust_in_tech_stack (49p), etc.
- 477 chunks embedded via supervisor (43/sec)
- HNSW index built in 0.44s
- Search results <200ms, finding correct conversations by meaning
- Tested: "S3 architecture", "CRM WordPress", "Rust vs Python" — all correct

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/vectord/src/chunker.rs

@@ -40,48 +40,45 @@ pub fn chunk_text(
         }];
     }
 
+    // Work with char indices, not byte indices, to handle Unicode safely
+    let chars: Vec<char> = text.chars().collect();
+    let char_len = chars.len();
     let mut chunks = Vec::new();
-    let mut start = 0;
+    let mut start = 0usize; // char index
     let mut idx = 0u32;
 
-    while start < text.len() {
+    while start < char_len {
-        let end = (start + chunk_size).min(text.len());
+        let end = (start + chunk_size).min(char_len);
+        let chunk_str: String = chars[start..end].iter().collect();
 
-        // Try to break at a sentence or paragraph boundary
+        let actual_end = if end < char_len {
-        let chunk_text = &text[start..end];
+            // Look for last sentence boundary
-        let actual_end = if end < text.len() {
+            if let Some(pos) = chunk_str.rfind(". ") {
-            // Look for last sentence boundary in the chunk
+                start + chunk_str[..pos + 2].chars().count()
-            if let Some(pos) = chunk_text.rfind(". ") {
+            } else if let Some(pos) = chunk_str.rfind('\n') {
-                start + pos + 2
+                start + chunk_str[..pos + 1].chars().count()
-            } else if let Some(pos) = chunk_text.rfind('\n') {
+            } else if let Some(pos) = chunk_str.rfind(' ') {
-                start + pos + 1
+                start + chunk_str[..pos + 1].chars().count()
-            } else {
-                // Fall back to word boundary
-                if let Some(pos) = chunk_text.rfind(' ') {
-                    start + pos + 1
             } else {
                 end
             }
-            }
         } else {
             end
         };
 
-        let chunk = text[start..actual_end].trim();
+        let chunk: String = chars[start..actual_end].iter().collect();
-        if !chunk.is_empty() {
+        let trimmed = chunk.trim();
+        if !trimmed.is_empty() {
             chunks.push(TextChunk {
                 source: source.to_string(),
                 doc_id: doc_id.to_string(),
                 chunk_idx: idx,
-                text: chunk.to_string(),
+                text: trimmed.to_string(),
             });
             idx += 1;
         }
 
-        // Advance with overlap
+        if actual_end >= char_len { break; }
-        if actual_end >= text.len() {
-            break;
-        }
         start = if actual_end > overlap { actual_end - overlap } else { actual_end };
     }
 

data/_catalog/manifests/0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1.json

@@ -0,0 +1,72 @@
+{
+  "id": "0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1",
+  "name": "rust_in_tech_stack",
+  "schema_fingerprint": "314ee8ede68287240d7cec1403d252787c8ceb4ee2afa72cffcb0853ac3ac889",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/rust_in_tech_stack.parquet",
+      "size_bytes": 99623,
+      "created_at": "2026-03-28T03:32:50.769271381Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:32:50.769272195Z",
+  "updated_at": "2026-03-28T03:32:50.769710478Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Rust in Tech Stack.pdf",
+    "ingest_job": "ingest-1774668770769",
+    "ingest_timestamp": "2026-03-28T03:32:50.769271381Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 49
+}

data/_catalog/manifests/54cc1ae9-14fe-40f6-88e4-7c623f57b577.json

@@ -0,0 +1,72 @@
+{
+  "id": "54cc1ae9-14fe-40f6-88e4-7c623f57b577",
+  "name": "crm_development_options",
+  "schema_fingerprint": "bb04328fa99161403fa0993f59bf14adcf08eb551c7b0c5e9b9e305e5e0c8556",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/crm_development_options.parquet",
+      "size_bytes": 50528,
+      "created_at": "2026-03-28T03:30:29.036504470Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:30:29.036504841Z",
+  "updated_at": "2026-03-28T03:30:29.036642992Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "CRM Development Options.pdf",
+    "ingest_job": "ingest-1774668629036",
+    "ingest_timestamp": "2026-03-28T03:30:29.036504470Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 25
+}

data/_catalog/manifests/5d4bbdb1-fb53-4cea-86d5-309fe067111c.json

@@ -0,0 +1,72 @@
+{
+  "id": "5d4bbdb1-fb53-4cea-86d5-309fe067111c",
+  "name": "legacy_staffing_crm_search",
+  "schema_fingerprint": "87b3fa58cae74468bf7fbbe32397b1a88bc281832bb0226b9eae5ce2501cda4e",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/legacy_staffing_crm_search.parquet",
+      "size_bytes": 2722188,
+      "created_at": "2026-03-28T03:34:14.444384147Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:34:14.444384964Z",
+  "updated_at": "2026-03-28T03:34:14.444631183Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Legacy Staffing CRM Search.pdf",
+    "ingest_job": "ingest-1774668854444",
+    "ingest_timestamp": "2026-03-28T03:34:14.444384147Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 1
+}

data/_catalog/manifests/7ca19535-838e-4f79-aa6f-d8ebaa2e6edc.json

@@ -0,0 +1,72 @@
+{
+  "id": "7ca19535-838e-4f79-aa6f-d8ebaa2e6edc",
+  "name": "code_sorting_assistance",
+  "schema_fingerprint": "47e317d38bbad23688f6c274cbecc3dfd7076e42a6a0d09002a0bef835789bc2",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/code_sorting_assistance.parquet",
+      "size_bytes": 122034,
+      "created_at": "2026-03-28T03:31:08.585681049Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:31:08.585682007Z",
+  "updated_at": "2026-03-28T03:31:08.585978498Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Code Sorting Assistance.pdf",
+    "ingest_job": "ingest-1774668668585",
+    "ingest_timestamp": "2026-03-28T03:31:08.585681049Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 70
+}

data/_catalog/manifests/8f5c82bf-93ec-4ce7-b2d1-61e559242add.json

@@ -0,0 +1,72 @@
+{
+  "id": "8f5c82bf-93ec-4ce7-b2d1-61e559242add",
+  "name": "llms3_update_prompt",
+  "schema_fingerprint": "77cfcdb5e22f095302bb816abd4e33380a6b357cd7d23efee5076083020fbf32",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/llms3_update_prompt.parquet",
+      "size_bytes": 39396,
+      "created_at": "2026-03-28T03:28:04.752204191Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:28:04.752204734Z",
+  "updated_at": "2026-03-28T03:28:04.752281926Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "LLMS3 Update Prompt.pdf",
+    "ingest_job": "ingest-1774668484752",
+    "ingest_timestamp": "2026-03-28T03:28:04.752204191Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 6
+}

data/_catalog/manifests/d609d775-1a8e-4114-8aee-3238017229a7.json

@@ -0,0 +1,72 @@
+{
+  "id": "d609d775-1a8e-4114-8aee-3238017229a7",
+  "name": "hybrid_indexing_setup",
+  "schema_fingerprint": "dfb09f4395d131e5fda5dab65f8c9dbe395d46d0e828f504e053cdb1cf23f24b",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/hybrid_indexing_setup.parquet",
+      "size_bytes": 131530,
+      "created_at": "2026-03-28T03:30:28.978970613Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:30:28.978971805Z",
+  "updated_at": "2026-03-28T03:30:28.979244113Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Hybrid Indexing Setup.pdf",
+    "ingest_job": "ingest-1774668628978",
+    "ingest_timestamp": "2026-03-28T03:30:28.978970613Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 69
+}

data/_catalog/manifests/f9055d87-c7c5-4109-b311-11fab418541f.json

@@ -0,0 +1,72 @@
+{
+  "id": "f9055d87-c7c5-4109-b311-11fab418541f",
+  "name": "spark_vs_flink_iceberg",
+  "schema_fingerprint": "e3f74df7c9db965c184ff5f5ec92b0f68168a7f6e367e8253d28fca571a11688",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/spark_vs_flink_iceberg.parquet",
+      "size_bytes": 163762,
+      "created_at": "2026-03-28T03:35:18.667429709Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:35:18.667430594Z",
+  "updated_at": "2026-03-28T03:35:18.667593640Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Spark vs Flink Iceberg.pdf",
+    "ingest_job": "ingest-1774668918667",
+    "ingest_timestamp": "2026-03-28T03:35:18.667429709Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 124
+}

data/_catalog/manifests/faef511e-4f34-4002-b2f4-8aecb8cfb80e.json

@@ -0,0 +1,72 @@
+{
+  "id": "faef511e-4f34-4002-b2f4-8aecb8cfb80e",
+  "name": "hybrid_ai_data_lake",
+  "schema_fingerprint": "1fa158168cf5f542e9cdeec30c9e9aad9b12285b66e6fbdb7321e5d9f7039515",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/hybrid_ai_data_lake.parquet",
+      "size_bytes": 220352,
+      "created_at": "2026-03-28T03:35:16.027063136Z"
+    }
+  ],
+  "created_at": "2026-03-28T03:35:16.027064491Z",
+  "updated_at": "2026-03-28T03:35:16.027333389Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "source_file",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "conversation_title",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "turn_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "role",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page_number",
+      "data_type": "Int32",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "pdf",
+    "source_file": "Hybrid AI Data Lake.pdf",
+    "ingest_job": "ingest-1774668916027",
+    "ingest_timestamp": "2026-03-28T03:35:16.027063136Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 133
+}

data/vectors/meta/observer_kb.json

@@ -0,0 +1,15 @@
+{
+  "index_name": "observer_kb",
+  "source": "chatgpt_pdfs",
+  "model_name": "nomic-embed-text",
+  "model_version": "latest",
+  "dimensions": 768,
+  "chunk_count": 477,
+  "doc_count": 477,
+  "chunk_size": 500,
+  "overlap": 50,
+  "storage_key": "vectors/observer_kb.parquet",
+  "created_at": "2026-03-28T03:52:23.689193948Z",
+  "build_time_secs": 11.022282,
+  "chunks_per_sec": 43.275978
+}

data/vectors/meta/observer_kb_test.json

@@ -0,0 +1,15 @@
+{
+  "index_name": "observer_kb_test",
+  "source": "chatgpt",
+  "model_name": "nomic-embed-text",
+  "model_version": "latest",
+  "dimensions": 768,
+  "chunk_count": 100,
+  "doc_count": 100,
+  "chunk_size": 500,
+  "overlap": 50,
+  "storage_key": "vectors/observer_kb_test.parquet",
+  "created_at": "2026-03-28T03:52:02.861946987Z",
+  "build_time_secs": 1.8791063,
+  "chunks_per_sec": 53.21679
+}

data/vectors/meta/observer_test.json

@@ -0,0 +1,15 @@
+{
+  "index_name": "observer_test",
+  "source": "test",
+  "model_name": "nomic-embed-text",
+  "model_version": "latest",
+  "dimensions": 768,
+  "chunk_count": 15,
+  "doc_count": 5,
+  "chunk_size": 500,
+  "overlap": 50,
+  "storage_key": "vectors/observer_test.parquet",
+  "created_at": "2026-03-28T03:46:22.891189902Z",
+  "build_time_secs": 2.113568,
+  "chunks_per_sec": 7.0970035
+}

data/vectors/meta/quick_test.json

@@ -0,0 +1,15 @@
+{
+  "index_name": "quick_test",
+  "source": "test",
+  "model_name": "nomic-embed-text",
+  "model_version": "latest",
+  "dimensions": 768,
+  "chunk_count": 5,
+  "doc_count": 5,
+  "chunk_size": 500,
+  "overlap": 50,
+  "storage_key": "vectors/quick_test.parquet",
+  "created_at": "2026-03-28T03:51:49.689474954Z",
+  "build_time_secs": 1.3572433,
+  "chunks_per_sec": 3.6839378
+}