Server-side pagination for large result sets
- ResultStore: execute query, store batches server-side, serve pages on demand
- POST /query/paged → returns query_id + total_rows + page count (no rows)
- GET /query/page/{id}/{page}?size=100 → returns one page of rows
- RecordBatch slicing for efficient page extraction from Arrow batches
- LRU eviction: keeps 50 most recent query results in memory
- Tested: 100K rows → 1,000 pages of 100, any page fetchable by number
- Supervisor pattern: chunk results, serve on demand, retry-safe (idempotent GET)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "6d0002ef-28de-4b92-abed-0bfb7ab5fb6f",
|
||||
"name": "candidates",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/candidates.parquet",
|
||||
"size_bytes": 10592165,
|
||||
"created_at": "2026-03-28T01:54:06.898505222Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:06.898506121Z",
|
||||
"updated_at": "2026-03-28T01:54:06.898506121Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "73a4ef71-5b2a-4124-a2c2-fcefdde99624",
|
||||
"name": "job_orders",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/job_orders.parquet",
|
||||
"size_bytes": 905534,
|
||||
"created_at": "2026-03-28T01:54:07.007332060Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:07.007332775Z",
|
||||
"updated_at": "2026-03-28T01:54:07.007332775Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "b56694bb-f4f7-447d-b4d6-ac89103a6a3a",
|
||||
"name": "call_log",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/call_log.parquet",
|
||||
"size_bytes": 35951077,
|
||||
"created_at": "2026-03-28T01:54:15.457573021Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:15.457573701Z",
|
||||
"updated_at": "2026-03-28T01:54:15.457573701Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "daa5a11b-6898-40ba-a811-3bd4b6f5b750",
|
||||
"name": "email_log",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/email_log.parquet",
|
||||
"size_bytes": 16768671,
|
||||
"created_at": "2026-03-28T01:54:17.356056562Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:17.356057529Z",
|
||||
"updated_at": "2026-03-28T01:54:17.356057529Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "dd396ee4-3a1a-4939-af60-84d409eebed4",
|
||||
"name": "clients",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/clients.parquet",
|
||||
"size_bytes": 21971,
|
||||
"created_at": "2026-03-28T01:54:06.904379300Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:06.904379793Z",
|
||||
"updated_at": "2026-03-28T01:54:06.904379793Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "f85285e3-111a-4a55-aa58-05fd740ed862",
|
||||
"name": "timesheets",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/timesheets.parquet",
|
||||
"size_bytes": 17539932,
|
||||
"created_at": "2026-03-28T01:54:11.947773535Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:11.947778065Z",
|
||||
"updated_at": "2026-03-28T01:54:11.947778065Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "ffa72582-dc11-4454-a46e-35839a7d04bb",
|
||||
"name": "placements",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/placements.parquet",
|
||||
"size_bytes": 1213820,
|
||||
"created_at": "2026-03-28T01:54:07.109218264Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:07.109219070Z",
|
||||
"updated_at": "2026-03-28T01:54:07.109219070Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user