diff --git a/crates/vectord/src/chunker.rs b/crates/vectord/src/chunker.rs index aed1a62..e71f98b 100644 --- a/crates/vectord/src/chunker.rs +++ b/crates/vectord/src/chunker.rs @@ -40,48 +40,45 @@ pub fn chunk_text( }]; } + // Work with char indices, not byte indices, to handle Unicode safely + let chars: Vec = text.chars().collect(); + let char_len = chars.len(); let mut chunks = Vec::new(); - let mut start = 0; + let mut start = 0usize; // char index let mut idx = 0u32; - while start < text.len() { - let end = (start + chunk_size).min(text.len()); + while start < char_len { + let end = (start + chunk_size).min(char_len); + let chunk_str: String = chars[start..end].iter().collect(); - // Try to break at a sentence or paragraph boundary - let chunk_text = &text[start..end]; - let actual_end = if end < text.len() { - // Look for last sentence boundary in the chunk - if let Some(pos) = chunk_text.rfind(". ") { - start + pos + 2 - } else if let Some(pos) = chunk_text.rfind('\n') { - start + pos + 1 + let actual_end = if end < char_len { + // Look for last sentence boundary + if let Some(pos) = chunk_str.rfind(". ") { + start + chunk_str[..pos + 2].chars().count() + } else if let Some(pos) = chunk_str.rfind('\n') { + start + chunk_str[..pos + 1].chars().count() + } else if let Some(pos) = chunk_str.rfind(' ') { + start + chunk_str[..pos + 1].chars().count() } else { - // Fall back to word boundary - if let Some(pos) = chunk_text.rfind(' ') { - start + pos + 1 - } else { - end - } + end } } else { end }; - let chunk = text[start..actual_end].trim(); - if !chunk.is_empty() { + let chunk: String = chars[start..actual_end].iter().collect(); + let trimmed = chunk.trim(); + if !trimmed.is_empty() { chunks.push(TextChunk { source: source.to_string(), doc_id: doc_id.to_string(), chunk_idx: idx, - text: chunk.to_string(), + text: trimmed.to_string(), }); idx += 1; } - // Advance with overlap - if actual_end >= text.len() { - break; - } + if actual_end >= char_len { break; } start = if actual_end > overlap { actual_end - overlap } else { actual_end }; } diff --git a/data/_catalog/manifests/0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1.json b/data/_catalog/manifests/0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1.json new file mode 100644 index 0000000..bd96fdc --- /dev/null +++ b/data/_catalog/manifests/0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1.json @@ -0,0 +1,72 @@ +{ + "id": "0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1", + "name": "rust_in_tech_stack", + "schema_fingerprint": "314ee8ede68287240d7cec1403d252787c8ceb4ee2afa72cffcb0853ac3ac889", + "objects": [ + { + "bucket": "data", + "key": "datasets/rust_in_tech_stack.parquet", + "size_bytes": 99623, + "created_at": "2026-03-28T03:32:50.769271381Z" + } + ], + "created_at": "2026-03-28T03:32:50.769272195Z", + "updated_at": "2026-03-28T03:32:50.769710478Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "Rust in Tech Stack.pdf", + "ingest_job": "ingest-1774668770769", + "ingest_timestamp": "2026-03-28T03:32:50.769271381Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 49 +} \ No newline at end of file diff --git a/data/_catalog/manifests/54cc1ae9-14fe-40f6-88e4-7c623f57b577.json b/data/_catalog/manifests/54cc1ae9-14fe-40f6-88e4-7c623f57b577.json new file mode 100644 index 0000000..7b0483e --- /dev/null +++ b/data/_catalog/manifests/54cc1ae9-14fe-40f6-88e4-7c623f57b577.json @@ -0,0 +1,72 @@ +{ + "id": "54cc1ae9-14fe-40f6-88e4-7c623f57b577", + "name": "crm_development_options", + "schema_fingerprint": "bb04328fa99161403fa0993f59bf14adcf08eb551c7b0c5e9b9e305e5e0c8556", + "objects": [ + { + "bucket": "data", + "key": "datasets/crm_development_options.parquet", + "size_bytes": 50528, + "created_at": "2026-03-28T03:30:29.036504470Z" + } + ], + "created_at": "2026-03-28T03:30:29.036504841Z", + "updated_at": "2026-03-28T03:30:29.036642992Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "CRM Development Options.pdf", + "ingest_job": "ingest-1774668629036", + "ingest_timestamp": "2026-03-28T03:30:29.036504470Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 25 +} \ No newline at end of file diff --git a/data/_catalog/manifests/5d4bbdb1-fb53-4cea-86d5-309fe067111c.json b/data/_catalog/manifests/5d4bbdb1-fb53-4cea-86d5-309fe067111c.json new file mode 100644 index 0000000..cdcaaf9 --- /dev/null +++ b/data/_catalog/manifests/5d4bbdb1-fb53-4cea-86d5-309fe067111c.json @@ -0,0 +1,72 @@ +{ + "id": "5d4bbdb1-fb53-4cea-86d5-309fe067111c", + "name": "legacy_staffing_crm_search", + "schema_fingerprint": "87b3fa58cae74468bf7fbbe32397b1a88bc281832bb0226b9eae5ce2501cda4e", + "objects": [ + { + "bucket": "data", + "key": "datasets/legacy_staffing_crm_search.parquet", + "size_bytes": 2722188, + "created_at": "2026-03-28T03:34:14.444384147Z" + } + ], + "created_at": "2026-03-28T03:34:14.444384964Z", + "updated_at": "2026-03-28T03:34:14.444631183Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "Legacy Staffing CRM Search.pdf", + "ingest_job": "ingest-1774668854444", + "ingest_timestamp": "2026-03-28T03:34:14.444384147Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 1 +} \ No newline at end of file diff --git a/data/_catalog/manifests/7ca19535-838e-4f79-aa6f-d8ebaa2e6edc.json b/data/_catalog/manifests/7ca19535-838e-4f79-aa6f-d8ebaa2e6edc.json new file mode 100644 index 0000000..1ea3d58 --- /dev/null +++ b/data/_catalog/manifests/7ca19535-838e-4f79-aa6f-d8ebaa2e6edc.json @@ -0,0 +1,72 @@ +{ + "id": "7ca19535-838e-4f79-aa6f-d8ebaa2e6edc", + "name": "code_sorting_assistance", + "schema_fingerprint": "47e317d38bbad23688f6c274cbecc3dfd7076e42a6a0d09002a0bef835789bc2", + "objects": [ + { + "bucket": "data", + "key": "datasets/code_sorting_assistance.parquet", + "size_bytes": 122034, + "created_at": "2026-03-28T03:31:08.585681049Z" + } + ], + "created_at": "2026-03-28T03:31:08.585682007Z", + "updated_at": "2026-03-28T03:31:08.585978498Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "Code Sorting Assistance.pdf", + "ingest_job": "ingest-1774668668585", + "ingest_timestamp": "2026-03-28T03:31:08.585681049Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 70 +} \ No newline at end of file diff --git a/data/_catalog/manifests/8f5c82bf-93ec-4ce7-b2d1-61e559242add.json b/data/_catalog/manifests/8f5c82bf-93ec-4ce7-b2d1-61e559242add.json new file mode 100644 index 0000000..922a790 --- /dev/null +++ b/data/_catalog/manifests/8f5c82bf-93ec-4ce7-b2d1-61e559242add.json @@ -0,0 +1,72 @@ +{ + "id": "8f5c82bf-93ec-4ce7-b2d1-61e559242add", + "name": "llms3_update_prompt", + "schema_fingerprint": "77cfcdb5e22f095302bb816abd4e33380a6b357cd7d23efee5076083020fbf32", + "objects": [ + { + "bucket": "data", + "key": "datasets/llms3_update_prompt.parquet", + "size_bytes": 39396, + "created_at": "2026-03-28T03:28:04.752204191Z" + } + ], + "created_at": "2026-03-28T03:28:04.752204734Z", + "updated_at": "2026-03-28T03:28:04.752281926Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "LLMS3 Update Prompt.pdf", + "ingest_job": "ingest-1774668484752", + "ingest_timestamp": "2026-03-28T03:28:04.752204191Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 6 +} \ No newline at end of file diff --git a/data/_catalog/manifests/d609d775-1a8e-4114-8aee-3238017229a7.json b/data/_catalog/manifests/d609d775-1a8e-4114-8aee-3238017229a7.json new file mode 100644 index 0000000..3c54192 --- /dev/null +++ b/data/_catalog/manifests/d609d775-1a8e-4114-8aee-3238017229a7.json @@ -0,0 +1,72 @@ +{ + "id": "d609d775-1a8e-4114-8aee-3238017229a7", + "name": "hybrid_indexing_setup", + "schema_fingerprint": "dfb09f4395d131e5fda5dab65f8c9dbe395d46d0e828f504e053cdb1cf23f24b", + "objects": [ + { + "bucket": "data", + "key": "datasets/hybrid_indexing_setup.parquet", + "size_bytes": 131530, + "created_at": "2026-03-28T03:30:28.978970613Z" + } + ], + "created_at": "2026-03-28T03:30:28.978971805Z", + "updated_at": "2026-03-28T03:30:28.979244113Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "Hybrid Indexing Setup.pdf", + "ingest_job": "ingest-1774668628978", + "ingest_timestamp": "2026-03-28T03:30:28.978970613Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 69 +} \ No newline at end of file diff --git a/data/_catalog/manifests/f9055d87-c7c5-4109-b311-11fab418541f.json b/data/_catalog/manifests/f9055d87-c7c5-4109-b311-11fab418541f.json new file mode 100644 index 0000000..616ee2b --- /dev/null +++ b/data/_catalog/manifests/f9055d87-c7c5-4109-b311-11fab418541f.json @@ -0,0 +1,72 @@ +{ + "id": "f9055d87-c7c5-4109-b311-11fab418541f", + "name": "spark_vs_flink_iceberg", + "schema_fingerprint": "e3f74df7c9db965c184ff5f5ec92b0f68168a7f6e367e8253d28fca571a11688", + "objects": [ + { + "bucket": "data", + "key": "datasets/spark_vs_flink_iceberg.parquet", + "size_bytes": 163762, + "created_at": "2026-03-28T03:35:18.667429709Z" + } + ], + "created_at": "2026-03-28T03:35:18.667430594Z", + "updated_at": "2026-03-28T03:35:18.667593640Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "Spark vs Flink Iceberg.pdf", + "ingest_job": "ingest-1774668918667", + "ingest_timestamp": "2026-03-28T03:35:18.667429709Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 124 +} \ No newline at end of file diff --git a/data/_catalog/manifests/faef511e-4f34-4002-b2f4-8aecb8cfb80e.json b/data/_catalog/manifests/faef511e-4f34-4002-b2f4-8aecb8cfb80e.json new file mode 100644 index 0000000..ee1f178 --- /dev/null +++ b/data/_catalog/manifests/faef511e-4f34-4002-b2f4-8aecb8cfb80e.json @@ -0,0 +1,72 @@ +{ + "id": "faef511e-4f34-4002-b2f4-8aecb8cfb80e", + "name": "hybrid_ai_data_lake", + "schema_fingerprint": "1fa158168cf5f542e9cdeec30c9e9aad9b12285b66e6fbdb7321e5d9f7039515", + "objects": [ + { + "bucket": "data", + "key": "datasets/hybrid_ai_data_lake.parquet", + "size_bytes": 220352, + "created_at": "2026-03-28T03:35:16.027063136Z" + } + ], + "created_at": "2026-03-28T03:35:16.027064491Z", + "updated_at": "2026-03-28T03:35:16.027333389Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "source_file", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "conversation_title", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "turn_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "role", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page_number", + "data_type": "Int32", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "pdf", + "source_file": "Hybrid AI Data Lake.pdf", + "ingest_job": "ingest-1774668916027", + "ingest_timestamp": "2026-03-28T03:35:16.027063136Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 133 +} \ No newline at end of file diff --git a/data/datasets/code_sorting_assistance.parquet b/data/datasets/code_sorting_assistance.parquet new file mode 100644 index 0000000..32c2e56 Binary files /dev/null and b/data/datasets/code_sorting_assistance.parquet differ diff --git a/data/datasets/crm_development_options.parquet b/data/datasets/crm_development_options.parquet new file mode 100644 index 0000000..dfaf373 Binary files /dev/null and b/data/datasets/crm_development_options.parquet differ diff --git a/data/datasets/hybrid_ai_data_lake.parquet b/data/datasets/hybrid_ai_data_lake.parquet new file mode 100644 index 0000000..0f0c651 Binary files /dev/null and b/data/datasets/hybrid_ai_data_lake.parquet differ diff --git a/data/datasets/hybrid_indexing_setup.parquet b/data/datasets/hybrid_indexing_setup.parquet new file mode 100644 index 0000000..ef1ce4e Binary files /dev/null and b/data/datasets/hybrid_indexing_setup.parquet differ diff --git a/data/datasets/legacy_staffing_crm_search.parquet b/data/datasets/legacy_staffing_crm_search.parquet new file mode 100644 index 0000000..a3be1ee Binary files /dev/null and b/data/datasets/legacy_staffing_crm_search.parquet differ diff --git a/data/datasets/llms3_update_prompt.parquet b/data/datasets/llms3_update_prompt.parquet new file mode 100644 index 0000000..07d03cc Binary files /dev/null and b/data/datasets/llms3_update_prompt.parquet differ diff --git a/data/datasets/rust_in_tech_stack.parquet b/data/datasets/rust_in_tech_stack.parquet new file mode 100644 index 0000000..e400f75 Binary files /dev/null and b/data/datasets/rust_in_tech_stack.parquet differ diff --git a/data/datasets/spark_vs_flink_iceberg.parquet b/data/datasets/spark_vs_flink_iceberg.parquet new file mode 100644 index 0000000..7835bb2 Binary files /dev/null and b/data/datasets/spark_vs_flink_iceberg.parquet differ diff --git a/data/vectors/meta/observer_kb.json b/data/vectors/meta/observer_kb.json new file mode 100644 index 0000000..ba9b3f8 --- /dev/null +++ b/data/vectors/meta/observer_kb.json @@ -0,0 +1,15 @@ +{ + "index_name": "observer_kb", + "source": "chatgpt_pdfs", + "model_name": "nomic-embed-text", + "model_version": "latest", + "dimensions": 768, + "chunk_count": 477, + "doc_count": 477, + "chunk_size": 500, + "overlap": 50, + "storage_key": "vectors/observer_kb.parquet", + "created_at": "2026-03-28T03:52:23.689193948Z", + "build_time_secs": 11.022282, + "chunks_per_sec": 43.275978 +} \ No newline at end of file diff --git a/data/vectors/meta/observer_kb_test.json b/data/vectors/meta/observer_kb_test.json new file mode 100644 index 0000000..3243486 --- /dev/null +++ b/data/vectors/meta/observer_kb_test.json @@ -0,0 +1,15 @@ +{ + "index_name": "observer_kb_test", + "source": "chatgpt", + "model_name": "nomic-embed-text", + "model_version": "latest", + "dimensions": 768, + "chunk_count": 100, + "doc_count": 100, + "chunk_size": 500, + "overlap": 50, + "storage_key": "vectors/observer_kb_test.parquet", + "created_at": "2026-03-28T03:52:02.861946987Z", + "build_time_secs": 1.8791063, + "chunks_per_sec": 53.21679 +} \ No newline at end of file diff --git a/data/vectors/meta/observer_test.json b/data/vectors/meta/observer_test.json new file mode 100644 index 0000000..b497682 --- /dev/null +++ b/data/vectors/meta/observer_test.json @@ -0,0 +1,15 @@ +{ + "index_name": "observer_test", + "source": "test", + "model_name": "nomic-embed-text", + "model_version": "latest", + "dimensions": 768, + "chunk_count": 15, + "doc_count": 5, + "chunk_size": 500, + "overlap": 50, + "storage_key": "vectors/observer_test.parquet", + "created_at": "2026-03-28T03:46:22.891189902Z", + "build_time_secs": 2.113568, + "chunks_per_sec": 7.0970035 +} \ No newline at end of file diff --git a/data/vectors/meta/quick_test.json b/data/vectors/meta/quick_test.json new file mode 100644 index 0000000..a25d3eb --- /dev/null +++ b/data/vectors/meta/quick_test.json @@ -0,0 +1,15 @@ +{ + "index_name": "quick_test", + "source": "test", + "model_name": "nomic-embed-text", + "model_version": "latest", + "dimensions": 768, + "chunk_count": 5, + "doc_count": 5, + "chunk_size": 500, + "overlap": 50, + "storage_key": "vectors/quick_test.parquet", + "created_at": "2026-03-28T03:51:49.689474954Z", + "build_time_secs": 1.3572433, + "chunks_per_sec": 3.6839378 +} \ No newline at end of file diff --git a/data/vectors/observer_kb.parquet b/data/vectors/observer_kb.parquet new file mode 100644 index 0000000..f6ebc5f Binary files /dev/null and b/data/vectors/observer_kb.parquet differ diff --git a/data/vectors/observer_kb_test.parquet b/data/vectors/observer_kb_test.parquet new file mode 100644 index 0000000..1a8a6e1 Binary files /dev/null and b/data/vectors/observer_kb_test.parquet differ diff --git a/data/vectors/observer_test.parquet b/data/vectors/observer_test.parquet new file mode 100644 index 0000000..fd78f2c Binary files /dev/null and b/data/vectors/observer_test.parquet differ diff --git a/data/vectors/quick_test.parquet b/data/vectors/quick_test.parquet new file mode 100644 index 0000000..fa54b16 Binary files /dev/null and b/data/vectors/quick_test.parquet differ