Observer KB: 477 pages from 8 ChatGPT PDFs indexed and searchable
- Fixed Unicode crash in chunker (ligatures like fi causing byte boundary panic) - Chunker now works with char indices, not byte indices - 8 PDFs ingested: hybrid_ai_data_lake (133p), spark_vs_flink_iceberg (124p), code_sorting (70p), hybrid_indexing (69p), rust_in_tech_stack (49p), etc. - 477 chunks embedded via supervisor (43/sec) - HNSW index built in 0.44s - Search results <200ms, finding correct conversations by meaning - Tested: "S3 architecture", "CRM WordPress", "Rust vs Python" — all correct Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -40,48 +40,45 @@ pub fn chunk_text(
|
||||
}];
|
||||
}
|
||||
|
||||
// Work with char indices, not byte indices, to handle Unicode safely
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let char_len = chars.len();
|
||||
let mut chunks = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut start = 0usize; // char index
|
||||
let mut idx = 0u32;
|
||||
|
||||
while start < text.len() {
|
||||
let end = (start + chunk_size).min(text.len());
|
||||
while start < char_len {
|
||||
let end = (start + chunk_size).min(char_len);
|
||||
let chunk_str: String = chars[start..end].iter().collect();
|
||||
|
||||
// Try to break at a sentence or paragraph boundary
|
||||
let chunk_text = &text[start..end];
|
||||
let actual_end = if end < text.len() {
|
||||
// Look for last sentence boundary in the chunk
|
||||
if let Some(pos) = chunk_text.rfind(". ") {
|
||||
start + pos + 2
|
||||
} else if let Some(pos) = chunk_text.rfind('\n') {
|
||||
start + pos + 1
|
||||
} else {
|
||||
// Fall back to word boundary
|
||||
if let Some(pos) = chunk_text.rfind(' ') {
|
||||
start + pos + 1
|
||||
let actual_end = if end < char_len {
|
||||
// Look for last sentence boundary
|
||||
if let Some(pos) = chunk_str.rfind(". ") {
|
||||
start + chunk_str[..pos + 2].chars().count()
|
||||
} else if let Some(pos) = chunk_str.rfind('\n') {
|
||||
start + chunk_str[..pos + 1].chars().count()
|
||||
} else if let Some(pos) = chunk_str.rfind(' ') {
|
||||
start + chunk_str[..pos + 1].chars().count()
|
||||
} else {
|
||||
end
|
||||
}
|
||||
}
|
||||
} else {
|
||||
end
|
||||
};
|
||||
|
||||
let chunk = text[start..actual_end].trim();
|
||||
if !chunk.is_empty() {
|
||||
let chunk: String = chars[start..actual_end].iter().collect();
|
||||
let trimmed = chunk.trim();
|
||||
if !trimmed.is_empty() {
|
||||
chunks.push(TextChunk {
|
||||
source: source.to_string(),
|
||||
doc_id: doc_id.to_string(),
|
||||
chunk_idx: idx,
|
||||
text: chunk.to_string(),
|
||||
text: trimmed.to_string(),
|
||||
});
|
||||
idx += 1;
|
||||
}
|
||||
|
||||
// Advance with overlap
|
||||
if actual_end >= text.len() {
|
||||
break;
|
||||
}
|
||||
if actual_end >= char_len { break; }
|
||||
start = if actual_end > overlap { actual_end - overlap } else { actual_end };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "0ae8f29b-bf70-4c9d-8b64-6ea7cb7f76a1",
|
||||
"name": "rust_in_tech_stack",
|
||||
"schema_fingerprint": "314ee8ede68287240d7cec1403d252787c8ceb4ee2afa72cffcb0853ac3ac889",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/rust_in_tech_stack.parquet",
|
||||
"size_bytes": 99623,
|
||||
"created_at": "2026-03-28T03:32:50.769271381Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:32:50.769272195Z",
|
||||
"updated_at": "2026-03-28T03:32:50.769710478Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "Rust in Tech Stack.pdf",
|
||||
"ingest_job": "ingest-1774668770769",
|
||||
"ingest_timestamp": "2026-03-28T03:32:50.769271381Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 49
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "54cc1ae9-14fe-40f6-88e4-7c623f57b577",
|
||||
"name": "crm_development_options",
|
||||
"schema_fingerprint": "bb04328fa99161403fa0993f59bf14adcf08eb551c7b0c5e9b9e305e5e0c8556",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/crm_development_options.parquet",
|
||||
"size_bytes": 50528,
|
||||
"created_at": "2026-03-28T03:30:29.036504470Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:30:29.036504841Z",
|
||||
"updated_at": "2026-03-28T03:30:29.036642992Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "CRM Development Options.pdf",
|
||||
"ingest_job": "ingest-1774668629036",
|
||||
"ingest_timestamp": "2026-03-28T03:30:29.036504470Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 25
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "5d4bbdb1-fb53-4cea-86d5-309fe067111c",
|
||||
"name": "legacy_staffing_crm_search",
|
||||
"schema_fingerprint": "87b3fa58cae74468bf7fbbe32397b1a88bc281832bb0226b9eae5ce2501cda4e",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/legacy_staffing_crm_search.parquet",
|
||||
"size_bytes": 2722188,
|
||||
"created_at": "2026-03-28T03:34:14.444384147Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:34:14.444384964Z",
|
||||
"updated_at": "2026-03-28T03:34:14.444631183Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "Legacy Staffing CRM Search.pdf",
|
||||
"ingest_job": "ingest-1774668854444",
|
||||
"ingest_timestamp": "2026-03-28T03:34:14.444384147Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 1
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "7ca19535-838e-4f79-aa6f-d8ebaa2e6edc",
|
||||
"name": "code_sorting_assistance",
|
||||
"schema_fingerprint": "47e317d38bbad23688f6c274cbecc3dfd7076e42a6a0d09002a0bef835789bc2",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/code_sorting_assistance.parquet",
|
||||
"size_bytes": 122034,
|
||||
"created_at": "2026-03-28T03:31:08.585681049Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:31:08.585682007Z",
|
||||
"updated_at": "2026-03-28T03:31:08.585978498Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "Code Sorting Assistance.pdf",
|
||||
"ingest_job": "ingest-1774668668585",
|
||||
"ingest_timestamp": "2026-03-28T03:31:08.585681049Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 70
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "8f5c82bf-93ec-4ce7-b2d1-61e559242add",
|
||||
"name": "llms3_update_prompt",
|
||||
"schema_fingerprint": "77cfcdb5e22f095302bb816abd4e33380a6b357cd7d23efee5076083020fbf32",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/llms3_update_prompt.parquet",
|
||||
"size_bytes": 39396,
|
||||
"created_at": "2026-03-28T03:28:04.752204191Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:28:04.752204734Z",
|
||||
"updated_at": "2026-03-28T03:28:04.752281926Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "LLMS3 Update Prompt.pdf",
|
||||
"ingest_job": "ingest-1774668484752",
|
||||
"ingest_timestamp": "2026-03-28T03:28:04.752204191Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 6
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "d609d775-1a8e-4114-8aee-3238017229a7",
|
||||
"name": "hybrid_indexing_setup",
|
||||
"schema_fingerprint": "dfb09f4395d131e5fda5dab65f8c9dbe395d46d0e828f504e053cdb1cf23f24b",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/hybrid_indexing_setup.parquet",
|
||||
"size_bytes": 131530,
|
||||
"created_at": "2026-03-28T03:30:28.978970613Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:30:28.978971805Z",
|
||||
"updated_at": "2026-03-28T03:30:28.979244113Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "Hybrid Indexing Setup.pdf",
|
||||
"ingest_job": "ingest-1774668628978",
|
||||
"ingest_timestamp": "2026-03-28T03:30:28.978970613Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 69
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "f9055d87-c7c5-4109-b311-11fab418541f",
|
||||
"name": "spark_vs_flink_iceberg",
|
||||
"schema_fingerprint": "e3f74df7c9db965c184ff5f5ec92b0f68168a7f6e367e8253d28fca571a11688",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/spark_vs_flink_iceberg.parquet",
|
||||
"size_bytes": 163762,
|
||||
"created_at": "2026-03-28T03:35:18.667429709Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:35:18.667430594Z",
|
||||
"updated_at": "2026-03-28T03:35:18.667593640Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "Spark vs Flink Iceberg.pdf",
|
||||
"ingest_job": "ingest-1774668918667",
|
||||
"ingest_timestamp": "2026-03-28T03:35:18.667429709Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 124
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"id": "faef511e-4f34-4002-b2f4-8aecb8cfb80e",
|
||||
"name": "hybrid_ai_data_lake",
|
||||
"schema_fingerprint": "1fa158168cf5f542e9cdeec30c9e9aad9b12285b66e6fbdb7321e5d9f7039515",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/hybrid_ai_data_lake.parquet",
|
||||
"size_bytes": 220352,
|
||||
"created_at": "2026-03-28T03:35:16.027063136Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T03:35:16.027064491Z",
|
||||
"updated_at": "2026-03-28T03:35:16.027333389Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [
|
||||
{
|
||||
"name": "source_file",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "conversation_title",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "turn_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"data_type": "Utf8",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
},
|
||||
{
|
||||
"name": "page_number",
|
||||
"data_type": "Int32",
|
||||
"sensitivity": null,
|
||||
"description": "",
|
||||
"is_pii": false
|
||||
}
|
||||
],
|
||||
"lineage": {
|
||||
"source_system": "pdf",
|
||||
"source_file": "Hybrid AI Data Lake.pdf",
|
||||
"ingest_job": "ingest-1774668916027",
|
||||
"ingest_timestamp": "2026-03-28T03:35:16.027063136Z",
|
||||
"parent_datasets": []
|
||||
},
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": 133
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"index_name": "observer_kb",
|
||||
"source": "chatgpt_pdfs",
|
||||
"model_name": "nomic-embed-text",
|
||||
"model_version": "latest",
|
||||
"dimensions": 768,
|
||||
"chunk_count": 477,
|
||||
"doc_count": 477,
|
||||
"chunk_size": 500,
|
||||
"overlap": 50,
|
||||
"storage_key": "vectors/observer_kb.parquet",
|
||||
"created_at": "2026-03-28T03:52:23.689193948Z",
|
||||
"build_time_secs": 11.022282,
|
||||
"chunks_per_sec": 43.275978
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"index_name": "observer_kb_test",
|
||||
"source": "chatgpt",
|
||||
"model_name": "nomic-embed-text",
|
||||
"model_version": "latest",
|
||||
"dimensions": 768,
|
||||
"chunk_count": 100,
|
||||
"doc_count": 100,
|
||||
"chunk_size": 500,
|
||||
"overlap": 50,
|
||||
"storage_key": "vectors/observer_kb_test.parquet",
|
||||
"created_at": "2026-03-28T03:52:02.861946987Z",
|
||||
"build_time_secs": 1.8791063,
|
||||
"chunks_per_sec": 53.21679
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"index_name": "observer_test",
|
||||
"source": "test",
|
||||
"model_name": "nomic-embed-text",
|
||||
"model_version": "latest",
|
||||
"dimensions": 768,
|
||||
"chunk_count": 15,
|
||||
"doc_count": 5,
|
||||
"chunk_size": 500,
|
||||
"overlap": 50,
|
||||
"storage_key": "vectors/observer_test.parquet",
|
||||
"created_at": "2026-03-28T03:46:22.891189902Z",
|
||||
"build_time_secs": 2.113568,
|
||||
"chunks_per_sec": 7.0970035
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"index_name": "quick_test",
|
||||
"source": "test",
|
||||
"model_name": "nomic-embed-text",
|
||||
"model_version": "latest",
|
||||
"dimensions": 768,
|
||||
"chunk_count": 5,
|
||||
"doc_count": 5,
|
||||
"chunk_size": 500,
|
||||
"overlap": 50,
|
||||
"storage_key": "vectors/quick_test.parquet",
|
||||
"created_at": "2026-03-28T03:51:49.689474954Z",
|
||||
"build_time_secs": 1.3572433,
|
||||
"chunks_per_sec": 3.6839378
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user