profit/lakehouse
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Phase 8: Hot cache + incremental delta updates

Browse Source 
- MemCache: LRU in-memory cache for hot datasets (configurable max, default 16GB)
  Pin/evict/stats endpoints: POST /query/cache/pin, /cache/evict, GET /cache/stats
- Delta store: append-only delta Parquet files for row-level updates
  Write deltas without rewriting base files, merge at query time
- Compaction: POST /query/compact merges deltas into base Parquet
- Query engine: checks cache first, falls back to Parquet, merges deltas
- Benchmarked on 2.47M rows:
  1M row JOIN: 854ms cold → 96ms hot (8.9x speedup)
  100K filter: 62ms cold → 21ms hot (3x speedup)
  1.1M rows cached in 408MB RAM

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 08:37:28 -05:00
co-authored by Claude Opus 4.6
parent eae51977ab
commit 6df904a03c
23 changed files with 518 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/_catalog/manifests/021ac283-883b-4b13-83ce-5395bacdc33a.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "021ac283-883b-4b13-83ce-5395bacdc33a",
"name": "clients",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/clients.parquet",
"size_bytes": 21971,
"created_at": "2026-03-27T13:15:18.000750302Z"
}
],
"created_at": "2026-03-27T13:15:18.000757845Z",
"updated_at": "2026-03-27T13:15:18.000757845Z"
}
data/_catalog/manifests/052cf81b-f5b6-4439-92d7-ecf09b24bd8b.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "052cf81b-f5b6-4439-92d7-ecf09b24bd8b",
"name": "candidates",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/candidates.parquet",
"size_bytes": 10592165,
"created_at": "2026-03-27T13:15:17.989860994Z"
}
],
"created_at": "2026-03-27T13:15:17.989869155Z",
"updated_at": "2026-03-27T13:15:17.989869155Z"
}
data/_catalog/manifests/18d22cdd-24b3-4a65-bdcb-6624753e5ab7.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "18d22cdd-24b3-4a65-bdcb-6624753e5ab7",
"name": "job_orders",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/job_orders.parquet",
"size_bytes": 905534,
"created_at": "2026-03-27T13:36:42.130140103Z"
}
],
"created_at": "2026-03-27T13:36:42.130146127Z",
"updated_at": "2026-03-27T13:36:42.130146127Z"
}
data/_catalog/manifests/47756b77-9a2e-476c-8249-9b971f95fb2d.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "47756b77-9a2e-476c-8249-9b971f95fb2d",
"name": "call_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/call_log.parquet",
"size_bytes": 35951077,
"created_at": "2026-03-27T13:15:26.607093971Z"
}
],
"created_at": "2026-03-27T13:15:26.607099665Z",
"updated_at": "2026-03-27T13:15:26.607099665Z"
}
data/_catalog/manifests/8fa7cb8b-ab6b-4e64-9384-d2480e79dd7c.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "8fa7cb8b-ab6b-4e64-9384-d2480e79dd7c",
"name": "clients",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/clients.parquet",
"size_bytes": 21971,
"created_at": "2026-03-27T13:36:42.025701092Z"
}
],
"created_at": "2026-03-27T13:36:42.025707574Z",
"updated_at": "2026-03-27T13:36:42.025707574Z"
}
data/_catalog/manifests/b5d49316-9c9b-4a57-8221-13b6dcda551a.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "b5d49316-9c9b-4a57-8221-13b6dcda551a",
"name": "placements",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/placements.parquet",
"size_bytes": 1213820,
"created_at": "2026-03-27T13:36:42.237756183Z"
}
],
"created_at": "2026-03-27T13:36:42.237762120Z",
"updated_at": "2026-03-27T13:36:42.237762120Z"
}
data/_catalog/manifests/c00465bd-c562-419a-b40e-c557ba9054bf.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "c00465bd-c562-419a-b40e-c557ba9054bf",
"name": "candidates",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/candidates.parquet",
"size_bytes": 10592165,
"created_at": "2026-03-27T13:36:42.018896280Z"
}
],
"created_at": "2026-03-27T13:36:42.018904245Z",
"updated_at": "2026-03-27T13:36:42.018904245Z"
}
data/_catalog/manifests/c0224239-a265-4b15-a1e2-ebbc96aee60c.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "c0224239-a265-4b15-a1e2-ebbc96aee60c",
"name": "email_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/email_log.parquet",
"size_bytes": 16768671,
"created_at": "2026-03-27T13:15:28.446541739Z"
}
],
"created_at": "2026-03-27T13:15:28.446547070Z",
"updated_at": "2026-03-27T13:15:28.446547070Z"
}
data/_catalog/manifests/c8c9d519-b8b5-4d04-ba2b-5acf53c41bc2.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "c8c9d519-b8b5-4d04-ba2b-5acf53c41bc2",
"name": "timesheets",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/timesheets.parquet",
"size_bytes": 17539932,
"created_at": "2026-03-27T13:15:23.111118100Z"
}
],
"created_at": "2026-03-27T13:15:23.111124272Z",
"updated_at": "2026-03-27T13:15:23.111124272Z"
}
data/_catalog/manifests/dcca449b-a2f6-4c1f-99b6-c69dcdbdd204.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "dcca449b-a2f6-4c1f-99b6-c69dcdbdd204",
"name": "placements",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/placements.parquet",
"size_bytes": 1213820,
"created_at": "2026-03-27T13:15:18.264258909Z"
}
],
"created_at": "2026-03-27T13:15:18.264266375Z",
"updated_at": "2026-03-27T13:15:18.264266375Z"
}
data/_catalog/manifests/dce14141-f679-481b-9b48-13438cbfe057.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "dce14141-f679-481b-9b48-13438cbfe057",
"name": "email_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/email_log.parquet",
"size_bytes": 16768671,
"created_at": "2026-03-27T13:36:52.383853471Z"
}
],
"created_at": "2026-03-27T13:36:52.383859356Z",
"updated_at": "2026-03-27T13:36:52.383859356Z"
}
data/_catalog/manifests/e0bcb8de-a2c1-4706-bf2d-73c1b989a70d.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "e0bcb8de-a2c1-4706-bf2d-73c1b989a70d",
"name": "timesheets",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/timesheets.parquet",
"size_bytes": 17539932,
"created_at": "2026-03-27T13:36:46.998375016Z"
}
],
"created_at": "2026-03-27T13:36:46.998383728Z",
"updated_at": "2026-03-27T13:36:46.998383728Z"
}
data/_catalog/manifests/e47b637f-31c6-4209-ab3c-557f8c67c812.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "e47b637f-31c6-4209-ab3c-557f8c67c812",
"name": "call_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/call_log.parquet",
"size_bytes": 35951077,
"created_at": "2026-03-27T13:36:50.546706609Z"
}
],
"created_at": "2026-03-27T13:36:50.546712358Z",
"updated_at": "2026-03-27T13:36:50.546712358Z"
}
data/_catalog/manifests/e8cc1ad2-114e-4441-a526-b8e6de10cb59.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "e8cc1ad2-114e-4441-a526-b8e6de10cb59",
"name": "job_orders",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/job_orders.parquet",
"size_bytes": 905534,
"created_at": "2026-03-27T13:15:18.114659931Z"
}
],
"created_at": "2026-03-27T13:15:18.114667579Z",
"updated_at": "2026-03-27T13:15:18.114667579Z"
}
data/vectors/resumes_10k.parquet
BIN
View File
Binary file not shown.