Phase 1: storage + catalog layer

- storaged: object_store backend (LocalFileSystem), PUT/GET/DELETE/LIST endpoints - shared: arrow_helpers with Parquet roundtrip + schema fingerprinting (2 tests) - catalogd: in-memory registry with write-ahead manifest persistence to object storage - catalogd: POST/GET /datasets, GET /datasets/by-name/{name} - gateway: wires storaged + catalogd with shared object_store state - Phase tracker updated: Phase 0 + Phase 1 gates passed Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 05:15:27 -05:00
parent a52ca841c6
commit 655b6c0b37
17 changed files with 2331 additions and 30 deletions
@@ -1 +1,2 @@
+pub mod registry;
 pub mod service;
@@ -0,0 +1,108 @@
+use shared::types::{DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use storaged::ops;
+use object_store::ObjectStore;
+
+const MANIFEST_PREFIX: &str = "_catalog/manifests";
+
+/// In-memory dataset registry backed by manifest persistence in object storage.
+#[derive(Clone)]
+pub struct Registry {
+    datasets: Arc<RwLock<HashMap<DatasetId, DatasetManifest>>>,
+    store: Arc<dyn ObjectStore>,
+}
+
+impl Registry {
+    pub fn new(store: Arc<dyn ObjectStore>) -> Self {
+        Self {
+            datasets: Arc::new(RwLock::new(HashMap::new())),
+            store,
+        }
+    }
+
+    /// Rebuild in-memory index from persisted manifests on startup.
+    pub async fn rebuild(&self) -> Result<usize, String> {
+        let keys = ops::list(&self.store, Some(MANIFEST_PREFIX)).await?;
+        let mut datasets = self.datasets.write().await;
+        datasets.clear();
+        for key in &keys {
+            let data = ops::get(&self.store, key).await?;
+            let manifest: DatasetManifest =
+                serde_json::from_slice(&data).map_err(|e| e.to_string())?;
+            datasets.insert(manifest.id.clone(), manifest);
+        }
+        let count = datasets.len();
+        tracing::info!("catalog rebuilt: {count} datasets loaded");
+        Ok(count)
+    }
+
+    /// Register a new dataset. Persists manifest to storage before updating memory.
+    pub async fn register(
+        &self,
+        name: String,
+        schema_fingerprint: SchemaFingerprint,
+        objects: Vec<ObjectRef>,
+    ) -> Result<DatasetManifest, String> {
+        let now = chrono::Utc::now();
+        let manifest = DatasetManifest {
+            id: DatasetId::new(),
+            name,
+            schema_fingerprint,
+            objects,
+            created_at: now,
+            updated_at: now,
+        };
+
+        // Write-ahead: persist before in-memory update
+        let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
+        let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?;
+        ops::put(&self.store, &manifest_key, json.into()).await?;
+
+        let mut datasets = self.datasets.write().await;
+        datasets.insert(manifest.id.clone(), manifest.clone());
+
+        tracing::info!("registered dataset: {} ({})", manifest.name, manifest.id);
+        Ok(manifest)
+    }
+
+    /// Get a dataset by ID.
+    pub async fn get(&self, id: &DatasetId) -> Option<DatasetManifest> {
+        let datasets = self.datasets.read().await;
+        datasets.get(id).cloned()
+    }
+
+    /// Get a dataset by name.
+    pub async fn get_by_name(&self, name: &str) -> Option<DatasetManifest> {
+        let datasets = self.datasets.read().await;
+        datasets.values().find(|d| d.name == name).cloned()
+    }
+
+    /// List all datasets.
+    pub async fn list(&self) -> Vec<DatasetManifest> {
+        let datasets = self.datasets.read().await;
+        datasets.values().cloned().collect()
+    }
+
+    /// Add objects to an existing dataset.
+    pub async fn add_objects(
+        &self,
+        id: &DatasetId,
+        new_objects: Vec<ObjectRef>,
+    ) -> Result<DatasetManifest, String> {
+        let mut datasets = self.datasets.write().await;
+        let manifest = datasets.get_mut(id).ok_or_else(|| format!("dataset not found: {id}"))?;
+
+        manifest.objects.extend(new_objects);
+        manifest.updated_at = chrono::Utc::now();
+
+        // Persist updated manifest
+        let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
+        let json = serde_json::to_vec_pretty(manifest).map_err(|e| e.to_string())?;
+        ops::put(&self.store, &manifest_key, json.into()).await?;
+
+        Ok(manifest.clone())
+    }
+}
@@ -1,9 +1,125 @@
-use axum::{Router, routing::get};
+use axum::{
+    Json, Router,
+    extract::{Path, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+};
+use serde::{Deserialize, Serialize};
+use shared::types::{DatasetId, ObjectRef, SchemaFingerprint};
+use uuid::Uuid;

-pub fn router() -> Router {
-    Router::new().route("/health", get(health))
+use crate::registry::Registry;
+
+pub fn router(registry: Registry) -> Router {
+    Router::new()
+        .route("/health", get(health))
+        .route("/datasets", post(create_dataset))
+        .route("/datasets", get(list_datasets))
+        .route("/datasets/{id}", get(get_dataset))
+        .route("/datasets/by-name/{name}", get(get_dataset_by_name))
+        .with_state(registry)
 }

 async fn health() -> &'static str {
    "catalogd ok"
 }
+
+#[derive(Deserialize)]
+struct CreateDatasetRequest {
+    name: String,
+    schema_fingerprint: String,
+    objects: Vec<ObjectRefRequest>,
+}
+
+#[derive(Deserialize)]
+struct ObjectRefRequest {
+    bucket: String,
+    key: String,
+    size_bytes: u64,
+}
+
+#[derive(Serialize)]
+struct DatasetResponse {
+    id: String,
+    name: String,
+    schema_fingerprint: String,
+    objects: Vec<ObjectRefResponse>,
+    created_at: String,
+    updated_at: String,
+}
+
+#[derive(Serialize)]
+struct ObjectRefResponse {
+    bucket: String,
+    key: String,
+    size_bytes: u64,
+    created_at: String,
+}
+
+impl From<&shared::types::DatasetManifest> for DatasetResponse {
+    fn from(m: &shared::types::DatasetManifest) -> Self {
+        Self {
+            id: m.id.to_string(),
+            name: m.name.clone(),
+            schema_fingerprint: m.schema_fingerprint.0.clone(),
+            objects: m.objects.iter().map(|o| ObjectRefResponse {
+                bucket: o.bucket.clone(),
+                key: o.key.clone(),
+                size_bytes: o.size_bytes,
+                created_at: o.created_at.to_rfc3339(),
+            }).collect(),
+            created_at: m.created_at.to_rfc3339(),
+            updated_at: m.updated_at.to_rfc3339(),
+        }
+    }
+}
+
+async fn create_dataset(
+    State(registry): State<Registry>,
+    Json(req): Json<CreateDatasetRequest>,
+) -> impl IntoResponse {
+    let now = chrono::Utc::now();
+    let objects: Vec<ObjectRef> = req.objects.into_iter().map(|o| ObjectRef {
+        bucket: o.bucket,
+        key: o.key,
+        size_bytes: o.size_bytes,
+        created_at: now,
+    }).collect();
+
+    match registry.register(req.name, SchemaFingerprint(req.schema_fingerprint), objects).await {
+        Ok(manifest) => {
+            let resp = DatasetResponse::from(&manifest);
+            Ok((StatusCode::CREATED, Json(resp)))
+        }
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn list_datasets(State(registry): State<Registry>) -> impl IntoResponse {
+    let datasets = registry.list().await;
+    let resp: Vec<DatasetResponse> = datasets.iter().map(DatasetResponse::from).collect();
+    Json(resp)
+}
+
+async fn get_dataset(
+    State(registry): State<Registry>,
+    Path(id): Path<String>,
+) -> impl IntoResponse {
+    let uuid = Uuid::parse_str(&id).map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
+    let dataset_id = DatasetId(uuid);
+    match registry.get(&dataset_id).await {
+        Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
+        None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {id}"))),
+    }
+}
+
+async fn get_dataset_by_name(
+    State(registry): State<Registry>,
+    Path(name): Path<String>,
+) -> impl IntoResponse {
+    match registry.get_by_name(&name).await {
+        Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
+        None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {name}"))),
+    }
+}