Server-side pagination for large result sets
- ResultStore: execute query, store batches server-side, serve pages on demand
- POST /query/paged → returns query_id + total_rows + page count (no rows)
- GET /query/page/{id}/{page}?size=100 → returns one page of rows
- RecordBatch slicing for efficient page extraction from Arrow batches
- LRU eviction: keeps 50 most recent query results in memory
- Tested: 100K rows → 1,000 pages of 100, any page fetchable by number
- Supervisor pattern: chunk results, serve on demand, retry-safe (idempotent GET)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "6d0002ef-28de-4b92-abed-0bfb7ab5fb6f",
|
||||
"name": "candidates",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/candidates.parquet",
|
||||
"size_bytes": 10592165,
|
||||
"created_at": "2026-03-28T01:54:06.898505222Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:06.898506121Z",
|
||||
"updated_at": "2026-03-28T01:54:06.898506121Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "73a4ef71-5b2a-4124-a2c2-fcefdde99624",
|
||||
"name": "job_orders",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/job_orders.parquet",
|
||||
"size_bytes": 905534,
|
||||
"created_at": "2026-03-28T01:54:07.007332060Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:07.007332775Z",
|
||||
"updated_at": "2026-03-28T01:54:07.007332775Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "b56694bb-f4f7-447d-b4d6-ac89103a6a3a",
|
||||
"name": "call_log",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/call_log.parquet",
|
||||
"size_bytes": 35951077,
|
||||
"created_at": "2026-03-28T01:54:15.457573021Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:15.457573701Z",
|
||||
"updated_at": "2026-03-28T01:54:15.457573701Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "daa5a11b-6898-40ba-a811-3bd4b6f5b750",
|
||||
"name": "email_log",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/email_log.parquet",
|
||||
"size_bytes": 16768671,
|
||||
"created_at": "2026-03-28T01:54:17.356056562Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:17.356057529Z",
|
||||
"updated_at": "2026-03-28T01:54:17.356057529Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "dd396ee4-3a1a-4939-af60-84d409eebed4",
|
||||
"name": "clients",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/clients.parquet",
|
||||
"size_bytes": 21971,
|
||||
"created_at": "2026-03-28T01:54:06.904379300Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:06.904379793Z",
|
||||
"updated_at": "2026-03-28T01:54:06.904379793Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "f85285e3-111a-4a55-aa58-05fd740ed862",
|
||||
"name": "timesheets",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/timesheets.parquet",
|
||||
"size_bytes": 17539932,
|
||||
"created_at": "2026-03-28T01:54:11.947773535Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:11.947778065Z",
|
||||
"updated_at": "2026-03-28T01:54:11.947778065Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "ffa72582-dc11-4454-a46e-35839a7d04bb",
|
||||
"name": "placements",
|
||||
"schema_fingerprint": "auto",
|
||||
"objects": [
|
||||
{
|
||||
"bucket": "data",
|
||||
"key": "datasets/placements.parquet",
|
||||
"size_bytes": 1213820,
|
||||
"created_at": "2026-03-28T01:54:07.109218264Z"
|
||||
}
|
||||
],
|
||||
"created_at": "2026-03-28T01:54:07.109219070Z",
|
||||
"updated_at": "2026-03-28T01:54:07.109219070Z",
|
||||
"description": "",
|
||||
"owner": "",
|
||||
"sensitivity": null,
|
||||
"columns": [],
|
||||
"lineage": null,
|
||||
"freshness": null,
|
||||
"tags": [],
|
||||
"row_count": null
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1003,8 +1003,8 @@ fn ResultsTable(response: QueryResponse) -> Element {
|
||||
if response.row_count == 0 {
|
||||
div { class: "empty-sm", "no rows returned" }
|
||||
} else if let Some(rows) = rows {
|
||||
if rows.len() > 200 {
|
||||
div { class: "results-info", "Showing first 200 of {response.row_count} rows" }
|
||||
if rows.len() > 500 {
|
||||
div { class: "results-info", "Showing first 500 of {response.row_count} rows (use SQL tab with LIMIT for larger)" }
|
||||
}
|
||||
div { class: "table-wrap",
|
||||
table {
|
||||
@@ -1016,7 +1016,7 @@ fn ResultsTable(response: QueryResponse) -> Element {
|
||||
}
|
||||
}
|
||||
tbody {
|
||||
for row in rows.iter().take(200) {
|
||||
for row in rows.iter().take(500) {
|
||||
tr {
|
||||
for col in response.columns.iter() {
|
||||
td { {format_cell(row.get(&col.name))} }
|
||||
|
||||
Reference in New Issue
Block a user