Scale test: 2.47M rows + 10K vector index benchmarked

Benchmarks on 128GB RAM server: - 100K candidate filter (skills+city+status): 257ms - 1M timesheet aggregation (revenue by client): 942ms - 800K call log cross-reference (cold leads): 642ms - Triple JOIN recruiter performance: 487ms - 500K email open rate aggregation: 259ms - COUNT all 2.47M rows: 84ms - 10K vector search (cosine similarity): ~450ms - Embedding throughput: 49 chunks/sec via Ollama - RAG correctly refuses to hallucinate when no match exists Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 08:31:37 -05:00
parent 26fc98c885
commit eae51977ab
24 changed files with 369 additions and 105 deletions
@@ -0,0 +1,15 @@
+{
+  "id": "021ac283-883b-4b13-83ce-5395bacdc33a",
+  "name": "clients",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/clients.parquet",
+      "size_bytes": 21971,
+      "created_at": "2026-03-27T13:15:18.000750302Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:18.000757845Z",
+  "updated_at": "2026-03-27T13:15:18.000757845Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "052cf81b-f5b6-4439-92d7-ecf09b24bd8b",
+  "name": "candidates",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/candidates.parquet",
+      "size_bytes": 10592165,
+      "created_at": "2026-03-27T13:15:17.989860994Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:17.989869155Z",
+  "updated_at": "2026-03-27T13:15:17.989869155Z"
+}
@@ -1,15 +0,0 @@
-{
-  "id": "0927b27a-80a9-4790-a34f-bda7ff176aac",
-  "name": "job_orders",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/job_orders.parquet",
-      "size_bytes": 225889,
-      "created_at": "2026-03-27T13:11:41.384341257Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:41.384344032Z",
-  "updated_at": "2026-03-27T13:11:41.384344032Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "0bf1eb1f-b182-4025-9b44-b8553e678bcf",
-  "name": "timesheets",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/timesheets.parquet",
-      "size_bytes": 2458229,
-      "created_at": "2026-03-27T13:11:42.084209718Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:42.084217486Z",
-  "updated_at": "2026-03-27T13:11:42.084217486Z"
-}
@@ -0,0 +1,15 @@
+{
+  "id": "47756b77-9a2e-476c-8249-9b971f95fb2d",
+  "name": "call_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/call_log.parquet",
+      "size_bytes": 35951077,
+      "created_at": "2026-03-27T13:15:26.607093971Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:26.607099665Z",
+  "updated_at": "2026-03-27T13:15:26.607099665Z"
+}
@@ -1,15 +0,0 @@
-{
-  "id": "4be87c74-10b4-463c-b69d-f20c9cd18ed7",
-  "name": "candidates",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/candidates.parquet",
-      "size_bytes": 2003395,
-      "created_at": "2026-03-27T13:11:41.341589905Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:41.341599187Z",
-  "updated_at": "2026-03-27T13:11:41.341599187Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "75bb6855-488b-4300-89c2-970871bd99cc",
-  "name": "email_log",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/email_log.parquet",
-      "size_bytes": 1873775,
-      "created_at": "2026-03-27T13:11:42.757205427Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:42.757211105Z",
-  "updated_at": "2026-03-27T13:11:42.757211105Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "ad393eee-ba0c-4338-9a8b-236bba3816ac",
-  "name": "placements",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/placements.parquet",
-      "size_bytes": 217395,
-      "created_at": "2026-03-27T13:11:41.433628136Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:41.433633927Z",
-  "updated_at": "2026-03-27T13:11:41.433633927Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "b334b1eb-d7a2-473f-a7fa-017b17de74bd",
-  "name": "clients",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/clients.parquet",
-      "size_bytes": 34228,
-      "created_at": "2026-03-27T13:11:41.350247882Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:41.350250705Z",
-  "updated_at": "2026-03-27T13:11:41.350250705Z"
-}
@@ -0,0 +1,15 @@
+{
+  "id": "c0224239-a265-4b15-a1e2-ebbc96aee60c",
+  "name": "email_log",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/email_log.parquet",
+      "size_bytes": 16768671,
+      "created_at": "2026-03-27T13:15:28.446541739Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:28.446547070Z",
+  "updated_at": "2026-03-27T13:15:28.446547070Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "c8c9d519-b8b5-4d04-ba2b-5acf53c41bc2",
+  "name": "timesheets",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/timesheets.parquet",
+      "size_bytes": 17539932,
+      "created_at": "2026-03-27T13:15:23.111118100Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:23.111124272Z",
+  "updated_at": "2026-03-27T13:15:23.111124272Z"
+}
@@ -0,0 +1,15 @@
+{
+  "id": "dcca449b-a2f6-4c1f-99b6-c69dcdbdd204",
+  "name": "placements",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/placements.parquet",
+      "size_bytes": 1213820,
+      "created_at": "2026-03-27T13:15:18.264258909Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:18.264266375Z",
+  "updated_at": "2026-03-27T13:15:18.264266375Z"
+}
@@ -1,15 +0,0 @@
-{
-  "id": "e015f0e2-51e4-4301-855d-76c54992c5b9",
-  "name": "call_log",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/call_log.parquet",
-      "size_bytes": 3276693,
-      "created_at": "2026-03-27T13:11:42.483220340Z"
-    }
-  ],
-  "created_at": "2026-03-27T13:11:42.483225870Z",
-  "updated_at": "2026-03-27T13:11:42.483225870Z"
-}
@@ -0,0 +1,15 @@
+{
+  "id": "e8cc1ad2-114e-4441-a526-b8e6de10cb59",
+  "name": "job_orders",
+  "schema_fingerprint": "auto",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/job_orders.parquet",
+      "size_bytes": 905534,
+      "created_at": "2026-03-27T13:15:18.114659931Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:15:18.114667579Z",
+  "updated_at": "2026-03-27T13:15:18.114667579Z"
+}