Database connector: PostgreSQL → Parquet import

- POST /ingest/postgres/tables — list all tables in a database
- POST /ingest/postgres/import — import table → Parquet → catalog → queryable
- Auto type mapping: int2/4/8 → Int, float4/8 → Float64, bool → Boolean,
  text/varchar/jsonb/timestamp → Utf8 (safe default per ADR-010)
- Auto PII detection + lineage on import
- Empty password support for trust auth
- Tested: imported lab_trials (40 rows, 10 cols) and threat_intel (20 rows, 30 cols)
  from local knowledge_base Postgres database — immediately queryable

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ingestd/src/service.rs

@@ -5,12 +5,16 @@ use axum::{
     response::IntoResponse,
     routing::{get, post},
 };
+use bytes::Bytes;
 use object_store::ObjectStore;
 use serde::Deserialize;
 use std::sync::Arc;
 
 use catalogd::registry::Registry;
-use crate::pipeline;
+use crate::{db_ingest, pipeline};
+use shared::arrow_helpers::record_batch_to_parquet;
+use shared::types::{ObjectRef, SchemaFingerprint};
+use storaged::ops;
 
 #[derive(Clone)]
 pub struct IngestState {
@@ -22,6 +26,8 @@ pub fn router(state: IngestState) -> Router {
     Router::new()
         .route("/health", get(health))
         .route("/file", post(ingest_file))
+        .route("/postgres/tables", post(list_pg_tables))
+        .route("/postgres/import", post(import_pg_table))
         .with_state(state)
 }
 
@@ -31,17 +37,14 @@ async fn health() -> &'static str {
 
 #[derive(Deserialize)]
 struct IngestQuery {
-    /// Override dataset name (otherwise derived from filename)
     name: Option<String>,
 }
 
-/// Upload a file for ingestion. Accepts multipart/form-data with a "file" field.
 async fn ingest_file(
     State(state): State<IngestState>,
     Query(query): Query<IngestQuery>,
     mut multipart: Multipart,
 ) -> impl IntoResponse {
-    // Read the first file field
     let field = match multipart.next_field().await {
         Ok(Some(f)) => f,
         Ok(None) => return Err((StatusCode::BAD_REQUEST, "no file uploaded".to_string())),
@@ -67,3 +70,121 @@ async fn ingest_file(
         Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
     }
 }
+
+// --- PostgreSQL Import ---
+
+/// List tables in a PostgreSQL database.
+async fn list_pg_tables(
+    Json(config): Json<db_ingest::DbConfig>,
+) -> impl IntoResponse {
+    match db_ingest::list_postgres_tables(&config).await {
+        Ok(tables) => Ok(Json(tables)),
+        Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct PgImportRequest {
+    #[serde(flatten)]
+    config: db_ingest::DbConfig,
+    table: String,
+    /// Override dataset name (defaults to table name)
+    dataset_name: Option<String>,
+    /// Max rows to import (None = all)
+    limit: Option<usize>,
+}
+
+/// Import a PostgreSQL table into the lakehouse.
+async fn import_pg_table(
+    State(state): State<IngestState>,
+    Json(req): Json<PgImportRequest>,
+) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
+    tracing::info!("importing postgres table '{}' from {}:{}/{}",
+        req.table, req.config.host, req.config.port, req.config.database);
+
+    // Import from Postgres
+    let (schema, batches, db_result) = db_ingest::import_postgres_table(
+        &req.config, &req.table, req.limit,
+    ).await.map_err(|e| (StatusCode::BAD_GATEWAY, e))?;
+
+    if batches.is_empty() || db_result.rows == 0 {
+        return Ok((StatusCode::OK, Json(serde_json::json!({
+            "table": req.table,
+            "rows": 0,
+            "message": "table is empty",
+        }))));
+    }
+
+    // Convert to Parquet
+    let mut all_parquet = Vec::new();
+    for batch in &batches {
+        let pq = record_batch_to_parquet(batch)
+            .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
+        all_parquet.extend_from_slice(&pq);
+    }
+
+    let dataset_name = req.dataset_name.unwrap_or_else(|| req.table.clone());
+    let storage_key = format!("datasets/{}.parquet", dataset_name);
+    let parquet_size = all_parquet.len() as u64;
+
+    // Store
+    ops::put(&state.store, &storage_key, Bytes::from(all_parquet))
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
+
+    // Register
+    let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
+    let now = chrono::Utc::now();
+    state.registry.register(
+        dataset_name.clone(),
+        SchemaFingerprint(schema_fp.0),
+        vec![ObjectRef {
+            bucket: "data".to_string(),
+            key: storage_key.clone(),
+            size_bytes: parquet_size,
+            created_at: now,
+        }],
+    ).await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
+
+    // Auto-populate metadata (PII detection, lineage)
+    let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
+    let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
+    let columns: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
+        let sens = shared::pii::detect_sensitivity(f.name());
+        shared::types::ColumnMeta {
+            name: f.name().clone(),
+            data_type: f.data_type().to_string(),
+            sensitivity: sens.clone(),
+            description: String::new(),
+            is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
+        }
+    }).collect();
+
+    let lineage = shared::types::Lineage {
+        source_system: "postgresql".to_string(),
+        source_file: format!("{}:{}/{}.{}", req.config.host, req.config.port, req.config.database, req.table),
+        ingest_job: format!("pg-import-{}", now.timestamp_millis()),
+        ingest_timestamp: now,
+        parent_datasets: vec![],
+    };
+
+    let _ = state.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
+        sensitivity,
+        columns: Some(columns),
+        lineage: Some(lineage),
+        row_count: Some(db_result.rows as u64),
+        ..Default::default()
+    }).await;
+
+    tracing::info!("imported '{}' from postgres: {} rows → {}", dataset_name, db_result.rows, storage_key);
+
+    Ok((StatusCode::CREATED, Json(serde_json::json!({
+        "dataset_name": dataset_name,
+        "table": req.table,
+        "rows": db_result.rows,
+        "columns": db_result.columns,
+        "schema": db_result.schema_detected,
+        "storage_key": storage_key,
+        "size_bytes": parquet_size,
+    }))))
+}