profit/lakehouse
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Background job system for embedding — fixes 100K timeout

Browse Source 
- JobTracker: create/update/complete/fail jobs with progress tracking
- POST /vectors/index now returns immediately with job_id (HTTP 202)
- Embedding runs in tokio::spawn background task
- GET /vectors/jobs/{id} returns live progress (chunks embedded, rate, ETA)
- GET /vectors/jobs lists all jobs
- Progress logged every 100 batches with chunks/sec and ETA
- 100K embedding job running successfully at 44 chunks/sec
- System stays responsive during embedding (queries in 23ms)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 09:03:07 -05:00
co-authored by Claude Opus 4.6
parent 354c9c4a04
commit 6a532cb248
22 changed files with 313 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/_catalog/manifests/03b65605-7cce-4a49-b338-4f19b0ff2ed5.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "03b65605-7cce-4a49-b338-4f19b0ff2ed5",
"name": "call_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/call_log.parquet",
"size_bytes": 35951077,
"created_at": "2026-03-27T14:00:44.377704982Z"
}
],
"created_at": "2026-03-27T14:00:44.377712082Z",
"updated_at": "2026-03-27T14:00:44.377712082Z"
}
data/_catalog/manifests/0e4feb1a-1421-46ac-8222-ba0f0bd6e13e.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "0e4feb1a-1421-46ac-8222-ba0f0bd6e13e",
"name": "email_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/email_log.parquet",
"size_bytes": 16768671,
"created_at": "2026-03-27T14:00:46.272499334Z"
}
],
"created_at": "2026-03-27T14:00:46.272507485Z",
"updated_at": "2026-03-27T14:00:46.272507485Z"
}
data/_catalog/manifests/142c4090-fd14-4065-8c06-d9721c14ec87.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "142c4090-fd14-4065-8c06-d9721c14ec87",
"name": "candidates",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/candidates.parquet",
"size_bytes": 10592165,
"created_at": "2026-03-27T13:43:21.924470705Z"
}
],
"created_at": "2026-03-27T13:43:21.924477421Z",
"updated_at": "2026-03-27T13:43:21.924477421Z"
}
data/_catalog/manifests/154cb8fe-5dcb-4d23-8ddb-c95b259757e9.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "154cb8fe-5dcb-4d23-8ddb-c95b259757e9",
"name": "timesheets",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/timesheets.parquet",
"size_bytes": 17539932,
"created_at": "2026-03-27T14:00:40.845373500Z"
}
],
"created_at": "2026-03-27T14:00:40.845380446Z",
"updated_at": "2026-03-27T14:00:40.845380446Z"
}
data/_catalog/manifests/1e7a1b8d-6211-46b5-b030-02ac76f92564.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "1e7a1b8d-6211-46b5-b030-02ac76f92564",
"name": "email_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/email_log.parquet",
"size_bytes": 16768671,
"created_at": "2026-03-27T13:43:32.341429856Z"
}
],
"created_at": "2026-03-27T13:43:32.341435388Z",
"updated_at": "2026-03-27T13:43:32.341435388Z"
}
data/_catalog/manifests/29c177bd-3728-428a-ab0f-95169aae1106.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "29c177bd-3728-428a-ab0f-95169aae1106",
"name": "timesheets",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/timesheets.parquet",
"size_bytes": 17539932,
"created_at": "2026-03-27T13:43:26.951181242Z"
}
],
"created_at": "2026-03-27T13:43:26.951188331Z",
"updated_at": "2026-03-27T13:43:26.951188331Z"
}
data/_catalog/manifests/812e7d9a-0f50-49c0-b121-4cf758c304d9.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "812e7d9a-0f50-49c0-b121-4cf758c304d9",
"name": "placements",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/placements.parquet",
"size_bytes": 1213820,
"created_at": "2026-03-27T13:43:22.173146233Z"
}
],
"created_at": "2026-03-27T13:43:22.173152301Z",
"updated_at": "2026-03-27T13:43:22.173152301Z"
}
data/_catalog/manifests/91413428-b4b1-44b3-bb8d-5cb326019879.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "91413428-b4b1-44b3-bb8d-5cb326019879",
"name": "job_orders",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/job_orders.parquet",
"size_bytes": 905534,
"created_at": "2026-03-27T13:43:22.036039453Z"
}
],
"created_at": "2026-03-27T13:43:22.036045131Z",
"updated_at": "2026-03-27T13:43:22.036045131Z"
}
data/_catalog/manifests/9bb57bf9-2c19-42ed-84f4-83fd3c52b94a.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "9bb57bf9-2c19-42ed-84f4-83fd3c52b94a",
"name": "clients",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/clients.parquet",
"size_bytes": 21971,
"created_at": "2026-03-27T13:43:21.933347525Z"
}
],
"created_at": "2026-03-27T13:43:21.933351887Z",
"updated_at": "2026-03-27T13:43:21.933351887Z"
}
data/_catalog/manifests/d2ce2995-9c60-49c9-9b41-197020cebaae.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "d2ce2995-9c60-49c9-9b41-197020cebaae",
"name": "placements",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/placements.parquet",
"size_bytes": 1213820,
"created_at": "2026-03-27T14:00:35.885543632Z"
}
],
"created_at": "2026-03-27T14:00:35.885550623Z",
"updated_at": "2026-03-27T14:00:35.885550623Z"
}
data/_catalog/manifests/d8170213-d6af-4478-ae23-59f06fda3165.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "d8170213-d6af-4478-ae23-59f06fda3165",
"name": "job_orders",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/job_orders.parquet",
"size_bytes": 905534,
"created_at": "2026-03-27T14:00:35.780022147Z"
}
],
"created_at": "2026-03-27T14:00:35.780029168Z",
"updated_at": "2026-03-27T14:00:35.780029168Z"
}
data/_catalog/manifests/e1607b56-a826-4826-845a-76918127c6bf.json
-15
View File
@@ -1,15 +0,0 @@
{
"id": "e1607b56-a826-4826-845a-76918127c6bf",
"name": "call_log",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/call_log.parquet",
"size_bytes": 35951077,
"created_at": "2026-03-27T13:43:30.485776088Z"
}
],
"created_at": "2026-03-27T13:43:30.485783579Z",
"updated_at": "2026-03-27T13:43:30.485783579Z"
}
data/_catalog/manifests/e26d3633-a341-4229-9819-f287d98b788a.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "e26d3633-a341-4229-9819-f287d98b788a",
"name": "candidates",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/candidates.parquet",
"size_bytes": 10592165,
"created_at": "2026-03-27T14:00:35.662150713Z"
}
],
"created_at": "2026-03-27T14:00:35.662162510Z",
"updated_at": "2026-03-27T14:00:35.662162510Z"
}
data/_catalog/manifests/e4b8441f-d729-4465-91fb-2ed5f481e65d.json
+15
View File
@@ -0,0 +1,15 @@
{
"id": "e4b8441f-d729-4465-91fb-2ed5f481e65d",
"name": "clients",
"schema_fingerprint": "auto",
"objects": [
{
"bucket": "data",
"key": "datasets/clients.parquet",
"size_bytes": 21971,
"created_at": "2026-03-27T14:00:35.670181596Z"
}
],
"created_at": "2026-03-27T14:00:35.670184688Z",
"updated_at": "2026-03-27T14:00:35.670184688Z"
}