Phase 6: Ingest pipeline — CSV, JSON, PDF, text file support

- ingestd crate: detect file type → parse → schema detection → Parquet → catalog - CSV: auto-detect column types (int, float, bool, string), handles $, %, commas Strips dollar signs from amounts, flexible row parsing, sanitized column names - JSON: array or newline-delimited, nested object flattening (a.b.c → a_b_c) - PDF: text extraction via lopdf, one row per page (source_file, page_number, text) - Text/SMS: line-based ingestion with line numbers - Dedup: SHA-256 content hash, re-ingest same file = no-op - Gateway: POST /ingest/file multipart upload, 256MB body limit - Schema detection per ADR-010: ambiguous types default to String - 12 unit tests passing (CSV parsing, JSON flattening, type inference, dedup) - Tested: messy CSV with missing data, dollar amounts, N/A values → queryable Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 08:07:31 -05:00
parent 6740a017c7
commit bb05c4412e
25 changed files with 1261 additions and 78 deletions
@@ -0,0 +1,69 @@
+use axum::{
+    Json, Router,
+    extract::{Multipart, Query, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use object_store::ObjectStore;
+use serde::Deserialize;
+use std::sync::Arc;
+
+use catalogd::registry::Registry;
+use crate::pipeline;
+
+#[derive(Clone)]
+pub struct IngestState {
+    pub store: Arc<dyn ObjectStore>,
+    pub registry: Registry,
+}
+
+pub fn router(state: IngestState) -> Router {
+    Router::new()
+        .route("/health", get(health))
+        .route("/file", post(ingest_file))
+        .with_state(state)
+}
+
+async fn health() -> &'static str {
+    "ingestd ok"
+}
+
+#[derive(Deserialize)]
+struct IngestQuery {
+    /// Override dataset name (otherwise derived from filename)
+    name: Option<String>,
+}
+
+/// Upload a file for ingestion. Accepts multipart/form-data with a "file" field.
+async fn ingest_file(
+    State(state): State<IngestState>,
+    Query(query): Query<IngestQuery>,
+    mut multipart: Multipart,
+) -> impl IntoResponse {
+    // Read the first file field
+    let field = match multipart.next_field().await {
+        Ok(Some(f)) => f,
+        Ok(None) => return Err((StatusCode::BAD_REQUEST, "no file uploaded".to_string())),
+        Err(e) => return Err((StatusCode::BAD_REQUEST, format!("multipart error: {e}"))),
+    };
+
+    let filename = field.file_name().unwrap_or("unknown").to_string();
+    let content = field.bytes().await
+        .map_err(|e| (StatusCode::BAD_REQUEST, format!("read error: {e}")))?;
+
+    tracing::info!("received file '{}' ({} bytes) for ingest", filename, content.len());
+
+    let dataset_name = query.name.as_deref();
+
+    match pipeline::ingest_file(&filename, &content, dataset_name, &state.store, &state.registry).await {
+        Ok(result) => {
+            if result.deduplicated {
+                Ok((StatusCode::OK, Json(result)))
+            } else {
+                Ok((StatusCode::CREATED, Json(result)))
+            }
+        }
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}