Observer KB: 477 pages from 8 ChatGPT PDFs indexed and searchable

- Fixed Unicode crash in chunker (ligatures like fi causing byte boundary panic)
- Chunker now works with char indices, not byte indices
- 8 PDFs ingested: hybrid_ai_data_lake (133p), spark_vs_flink_iceberg (124p),
  code_sorting (70p), hybrid_indexing (69p), rust_in_tech_stack (49p), etc.
- 477 chunks embedded via supervisor (43/sec)
- HNSW index built in 0.44s
- Search results <200ms, finding correct conversations by meaning
- Tested: "S3 architecture", "CRM WordPress", "Rust vs Python" — all correct

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/vectord/src/chunker.rs

@@ -40,48 +40,45 @@ pub fn chunk_text(
         }];
     }
 
+    // Work with char indices, not byte indices, to handle Unicode safely
+    let chars: Vec<char> = text.chars().collect();
+    let char_len = chars.len();
     let mut chunks = Vec::new();
-    let mut start = 0;
+    let mut start = 0usize; // char index
     let mut idx = 0u32;
 
-    while start < text.len() {
-        let end = (start + chunk_size).min(text.len());
+    while start < char_len {
+        let end = (start + chunk_size).min(char_len);
+        let chunk_str: String = chars[start..end].iter().collect();
 
-        // Try to break at a sentence or paragraph boundary
-        let chunk_text = &text[start..end];
-        let actual_end = if end < text.len() {
-            // Look for last sentence boundary in the chunk
-            if let Some(pos) = chunk_text.rfind(". ") {
-                start + pos + 2
-            } else if let Some(pos) = chunk_text.rfind('\n') {
-                start + pos + 1
+        let actual_end = if end < char_len {
+            // Look for last sentence boundary
+            if let Some(pos) = chunk_str.rfind(". ") {
+                start + chunk_str[..pos + 2].chars().count()
+            } else if let Some(pos) = chunk_str.rfind('\n') {
+                start + chunk_str[..pos + 1].chars().count()
+            } else if let Some(pos) = chunk_str.rfind(' ') {
+                start + chunk_str[..pos + 1].chars().count()
             } else {
-                // Fall back to word boundary
-                if let Some(pos) = chunk_text.rfind(' ') {
-                    start + pos + 1
-                } else {
-                    end
-                }
+                end
             }
         } else {
             end
         };
 
-        let chunk = text[start..actual_end].trim();
-        if !chunk.is_empty() {
+        let chunk: String = chars[start..actual_end].iter().collect();
+        let trimmed = chunk.trim();
+        if !trimmed.is_empty() {
             chunks.push(TextChunk {
                 source: source.to_string(),
                 doc_id: doc_id.to_string(),
                 chunk_idx: idx,
-                text: chunk.to_string(),
+                text: trimmed.to_string(),
             });
             idx += 1;
         }
 
-        // Advance with overlap
-        if actual_end >= text.len() {
-            break;
-        }
+        if actual_end >= char_len { break; }
         start = if actual_end > overlap { actual_end - overlap } else { actual_end };
     }