Phase 6: Ingest pipeline — CSV, JSON, PDF, text file support

- ingestd crate: detect file type → parse → schema detection → Parquet → catalog - CSV: auto-detect column types (int, float, bool, string), handles $, %, commas Strips dollar signs from amounts, flexible row parsing, sanitized column names - JSON: array or newline-delimited, nested object flattening (a.b.c → a_b_c) - PDF: text extraction via lopdf, one row per page (source_file, page_number, text) - Text/SMS: line-based ingestion with line numbers - Dedup: SHA-256 content hash, re-ingest same file = no-op - Gateway: POST /ingest/file multipart upload, 256MB body limit - Schema detection per ADR-010: ambiguous types default to String - 12 unit tests passing (CSV parsing, JSON flattening, type inference, dedup) - Tested: messy CSV with missing data, dollar amounts, N/A values → queryable Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 08:07:31 -05:00
parent 6740a017c7
commit bb05c4412e
25 changed files with 1261 additions and 78 deletions
@@ -0,0 +1,15 @@
+{
+  "id": "1ca61945-d151-490b-81fd-2ca0397b68fa",
+  "name": "sms_messages",
+  "schema_fingerprint": "e1d079cbb2b7eedae5019767a886bd9a3396e291aa03630b9db69e9864948c09",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/sms_messages.parquet",
+      "size_bytes": 2018,
+      "created_at": "2026-03-27T13:07:14.253881797Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:07:14.253886027Z",
+  "updated_at": "2026-03-27T13:07:14.253886027Z"
+}
@@ -1,15 +0,0 @@
-{
-  "id": "1f81445a-404f-48ea-bd72-00f6721ee18b",
-  "name": "employees",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/employees.parquet",
-      "size_bytes": 1424,
-      "created_at": "2026-03-27T11:55:55.013996293Z"
-    }
-  ],
-  "created_at": "2026-03-27T11:55:55.014010258Z",
-  "updated_at": "2026-03-27T11:55:55.014010258Z"
-}
@@ -0,0 +1,15 @@
+{
+  "id": "478072c3-0c95-46a2-9193-f4b3ac4085ab",
+  "name": "test_ingest",
+  "schema_fingerprint": "4bdc4e5baeddc1187aecd4bfb788654f26145c2ba346b4bec6ca8ab950e1c133",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/test_ingest.parquet",
+      "size_bytes": 3129,
+      "created_at": "2026-03-27T13:06:57.437484309Z"
+    }
+  ],
+  "created_at": "2026-03-27T13:06:57.437488259Z",
+  "updated_at": "2026-03-27T13:06:57.437488259Z"
+}
@@ -1,15 +0,0 @@
-{
-  "id": "50d402a0-6fb4-4849-ac75-3541b8530aba",
-  "name": "u5",
-  "schema_fingerprint": "from-ui",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/events.parquet",
-      "size_bytes": 0,
-      "created_at": "2026-03-27T12:16:57.520054490Z"
-    }
-  ],
-  "created_at": "2026-03-27T12:16:57.520066991Z",
-  "updated_at": "2026-03-27T12:16:57.520066991Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "6c8dabf2-a6b3-4e23-b7d7-dc3b37a5e1a8",
-  "name": "events",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/events.parquet",
-      "size_bytes": 1127,
-      "created_at": "2026-03-27T11:55:55.017003417Z"
-    }
-  ],
-  "created_at": "2026-03-27T11:55:55.017011224Z",
-  "updated_at": "2026-03-27T11:55:55.017011224Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "f5281277-afe9-456b-b5fa-d667c2b0f41f",
-  "name": "products",
-  "schema_fingerprint": "auto",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/products.parquet",
-      "size_bytes": 1341,
-      "created_at": "2026-03-27T11:55:55.019096056Z"
-    }
-  ],
-  "created_at": "2026-03-27T11:55:55.019098965Z",
-  "updated_at": "2026-03-27T11:55:55.019098965Z"
-}
@@ -1,15 +0,0 @@
-{
-  "id": "fc046131-8f44-428a-af90-ebf6347d4cc6",
-  "name": "u",
-  "schema_fingerprint": "from-ui",
-  "objects": [
-    {
-      "bucket": "data",
-      "key": "datasets/employees.parquet",
-      "size_bytes": 0,
-      "created_at": "2026-03-27T12:16:51.462058823Z"
-    }
-  ],
-  "created_at": "2026-03-27T12:16:51.462071311Z",
-  "updated_at": "2026-03-27T12:16:51.462071311Z"
-}