Federation layer 2: header routing + cross-bucket SQL

Three pieces of the multi-bucket federation made real:

1. Catalog migration (POST /catalog/migrate-buckets)
   - One-shot normalizer for ObjectRef.bucket field
   - Empty -> "primary"; legacy "data"/"local" -> "primary"
   - Idempotent; re-running on canonical state is no-op
   - Ran on existing catalog: 12 refs renamed from "data", 2 already
     "primary", all 14 now canonical

2. X-Lakehouse-Bucket header middleware on ingest
   - resolve_bucket() helper extracts header, returns
     (bucket_name, store) or 404 with valid bucket list
   - ingest_file and ingest_db_stream now route writes per-request
   - Defaults to "primary" when header absent
   - pipeline::ingest_file_to_bucket records the actual bucket on the
     ObjectRef so catalog stays the source of truth for "where does this
     data live"
   - Verified: ingest with X-Lakehouse-Bucket: testing lands in
     data/_testing/, ingest without header lands in data/, bad header
     returns 404 with hint

3. queryd registers every bucket with DataFusion
   - QueryEngine now holds Arc<BucketRegistry> instead of single store
   - build_context iterates all buckets, registers each as a separate
     ObjectStore under URL scheme "lakehouse-{bucket}://"
   - ListingTable URLs include the per-object bucket scheme so
     DataFusion routes scans automatically based on ObjectRef.bucket
   - Profile bucket names like "profile:user" sanitized to
     "lakehouse-profile-user" since URL host segments can't contain ":"
   - Tolerant of duplicate manifest entries (pre-existing
     pipeline::ingest_file behavior creates a fresh dataset id per
     ingest); duplicates skipped with debug log
   - Backward compat: legacy "lakehouse://data/" URL still registered
     pointing at primary

Success gate: cross-bucket CROSS JOIN
  SELECT p.name, p.role, a.species
  FROM people_test p          (bucket: testing)
  CROSS JOIN animals a        (bucket: primary)
  LIMIT 5
returns rows correctly. DataFusion routed each scan to its bucket's
ObjectStore based on the URL scheme.

No regressions: SELECT COUNT(*) FROM candidates still returns 100000
from the primary bucket.

Deferred to Phase 17:
- POST /profile/{user}/activate (HNSW hot-load on profile switch)
- vectord storage paths becoming bucket-scoped (trial journals,
  eval sets per-profile)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ingestd/src/pipeline.rs

@@ -28,12 +28,29 @@ pub struct IngestResult {
 }
 
 /// Full ingest pipeline: detect → parse → dedup → store → register.
+///
+/// Bucket name (federation layer 2) is recorded on the ObjectRef so the
+/// catalog knows which bucket holds the data — defaults to "primary" via
+/// the ingest service's resolve_bucket helper.
 pub async fn ingest_file(
     filename: &str,
     content: &[u8],
     dataset_name: Option<&str>,
     store: &Arc<dyn ObjectStore>,
     registry: &Registry,
+) -> Result<IngestResult, String> {
+    ingest_file_to_bucket(filename, content, dataset_name, "primary", store, registry).await
+}
+
+/// Same as `ingest_file` but with explicit target bucket name on the
+/// ObjectRef. Header-aware ingest endpoints call this directly.
+pub async fn ingest_file_to_bucket(
+    filename: &str,
+    content: &[u8],
+    dataset_name: Option<&str>,
+    bucket: &str,
+    store: &Arc<dyn ObjectStore>,
+    registry: &Registry,
 ) -> Result<IngestResult, String> {
     // 1. Detect file type
     let file_type = detect_file_type(filename, content);
@@ -98,7 +115,7 @@ pub async fn ingest_file(
     let schema_fp = fingerprint_schema(&schema);
     let now = chrono::Utc::now();
     let obj_ref = ObjectRef {
-        bucket: "data".to_string(),
+        bucket: bucket.to_string(),
         key: storage_key.clone(),
         size_bytes: parquet_size,
         created_at: now,

crates/ingestd/src/service.rs

@@ -1,7 +1,7 @@
 use axum::{
     Json, Router,
     extract::{Multipart, Query, State},
-    http::StatusCode,
+    http::{HeaderMap, StatusCode},
     response::IntoResponse,
     routing::{get, post},
 };
@@ -15,11 +15,37 @@ use crate::{db_ingest, pg_stream, pipeline};
 use shared::arrow_helpers::record_batch_to_parquet;
 use shared::types::{ObjectRef, SchemaFingerprint};
 use storaged::ops;
+use storaged::registry::BucketRegistry;
 
 #[derive(Clone)]
 pub struct IngestState {
     pub store: Arc<dyn ObjectStore>,
     pub registry: Registry,
+    /// Federation layer 2: lookup target bucket from request headers.
+    pub buckets: Arc<BucketRegistry>,
+}
+
+/// Resolve the target bucket from `X-Lakehouse-Bucket` header.
+/// Returns `(bucket_name, store_for_writes)`. Falls back to "primary"
+/// when the header is absent. Returns Err with the canonical bucket
+/// list when the header names an unknown bucket.
+fn resolve_bucket(
+    headers: &HeaderMap,
+    buckets: &BucketRegistry,
+) -> Result<(String, Arc<dyn ObjectStore>), (StatusCode, String)> {
+    let target = headers
+        .get("x-lakehouse-bucket")
+        .and_then(|h| h.to_str().ok())
+        .map(|s| s.to_string())
+        .unwrap_or_else(|| buckets.default_name().to_string());
+
+    match buckets.get(&target) {
+        Ok(store) => Ok((target, store)),
+        Err(_) => Err((
+            StatusCode::NOT_FOUND,
+            format!("unknown bucket '{}' — use GET /storage/buckets to list", target),
+        )),
+    }
 }
 
 pub fn router(state: IngestState) -> Router {
@@ -44,8 +70,11 @@ struct IngestQuery {
 async fn ingest_file(
     State(state): State<IngestState>,
     Query(query): Query<IngestQuery>,
+    headers: HeaderMap,
     mut multipart: Multipart,
 ) -> impl IntoResponse {
+    let (bucket, store) = resolve_bucket(&headers, &state.buckets)?;
+
     let field = match multipart.next_field().await {
         Ok(Some(f)) => f,
         Ok(None) => return Err((StatusCode::BAD_REQUEST, "no file uploaded".to_string())),
@@ -56,11 +85,14 @@ async fn ingest_file(
     let content = field.bytes().await
         .map_err(|e| (StatusCode::BAD_REQUEST, format!("read error: {e}")))?;
 
-    tracing::info!("received file '{}' ({} bytes) for ingest", filename, content.len());
+    tracing::info!(
+        "ingest '{}' ({} bytes) -> bucket={}",
+        filename, content.len(), bucket,
+    );
 
     let dataset_name = query.name.as_deref();
 
-    match pipeline::ingest_file(&filename, &content, dataset_name, &state.store, &state.registry).await {
+    match pipeline::ingest_file_to_bucket(&filename, &content, dataset_name, &bucket, &store, &state.registry).await {
         Ok(result) => {
             if result.deduplicated {
                 Ok((StatusCode::OK, Json(result)))
@@ -195,11 +227,13 @@ async fn import_pg_table(
 /// `POST /ingest/db` with `{dsn, table, dataset_name}`.
 async fn ingest_db_stream(
     State(state): State<IngestState>,
+    headers: HeaderMap,
     Json(req): Json<pg_stream::PgStreamRequest>,
 ) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
+    let (bucket, store) = resolve_bucket(&headers, &state.buckets)?;
     tracing::info!(
-        "pg stream ingest: table='{}' dataset='{:?}' batch_size={:?}",
-        req.table, req.dataset_name, req.batch_size,
+        "pg stream ingest: table='{}' dataset='{:?}' bucket='{}' batch_size={:?}",
+        req.table, req.dataset_name, bucket, req.batch_size,
     );
 
     // Stream from Postgres into Parquet bytes.
@@ -224,7 +258,7 @@ async fn ingest_db_stream(
     let storage_key = format!("datasets/{}.parquet", dataset_name);
     let size_bytes = parquet.len() as u64;
 
-    ops::put(&state.store, &storage_key, parquet)
+    ops::put(&store, &storage_key, parquet)
         .await
         .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
 
@@ -234,7 +268,7 @@ async fn ingest_db_stream(
         dataset_name.clone(),
         SchemaFingerprint(schema_fp.0),
         vec![ObjectRef {
-            bucket: "primary".to_string(),
+            bucket: bucket.clone(),
             key: storage_key.clone(),
             size_bytes,
             created_at: now,