Phase 1: storage + catalog layer

- storaged: object_store backend (LocalFileSystem), PUT/GET/DELETE/LIST endpoints
- shared: arrow_helpers with Parquet roundtrip + schema fingerprinting (2 tests)
- catalogd: in-memory registry with write-ahead manifest persistence to object storage
- catalogd: POST/GET /datasets, GET /datasets/by-name/{name}
- gateway: wires storaged + catalogd with shared object_store state
- Phase tracker updated: Phase 0 + Phase 1 gates passed

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 05:15:27 -05:00
co-authored by Claude Opus 4.6
parent a52ca841c6
commit 655b6c0b37
17 changed files with 2331 additions and 30 deletions
+108
View File
@@ -0,0 +1,108 @@
use shared::types::{DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use storaged::ops;
use object_store::ObjectStore;
const MANIFEST_PREFIX: &str = "_catalog/manifests";
/// In-memory dataset registry backed by manifest persistence in object storage.
#[derive(Clone)]
pub struct Registry {
datasets: Arc<RwLock<HashMap<DatasetId, DatasetManifest>>>,
store: Arc<dyn ObjectStore>,
}
impl Registry {
pub fn new(store: Arc<dyn ObjectStore>) -> Self {
Self {
datasets: Arc::new(RwLock::new(HashMap::new())),
store,
}
}
/// Rebuild in-memory index from persisted manifests on startup.
pub async fn rebuild(&self) -> Result<usize, String> {
let keys = ops::list(&self.store, Some(MANIFEST_PREFIX)).await?;
let mut datasets = self.datasets.write().await;
datasets.clear();
for key in &keys {
let data = ops::get(&self.store, key).await?;
let manifest: DatasetManifest =
serde_json::from_slice(&data).map_err(|e| e.to_string())?;
datasets.insert(manifest.id.clone(), manifest);
}
let count = datasets.len();
tracing::info!("catalog rebuilt: {count} datasets loaded");
Ok(count)
}
/// Register a new dataset. Persists manifest to storage before updating memory.
pub async fn register(
&self,
name: String,
schema_fingerprint: SchemaFingerprint,
objects: Vec<ObjectRef>,
) -> Result<DatasetManifest, String> {
let now = chrono::Utc::now();
let manifest = DatasetManifest {
id: DatasetId::new(),
name,
schema_fingerprint,
objects,
created_at: now,
updated_at: now,
};
// Write-ahead: persist before in-memory update
let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?;
ops::put(&self.store, &manifest_key, json.into()).await?;
let mut datasets = self.datasets.write().await;
datasets.insert(manifest.id.clone(), manifest.clone());
tracing::info!("registered dataset: {} ({})", manifest.name, manifest.id);
Ok(manifest)
}
/// Get a dataset by ID.
pub async fn get(&self, id: &DatasetId) -> Option<DatasetManifest> {
let datasets = self.datasets.read().await;
datasets.get(id).cloned()
}
/// Get a dataset by name.
pub async fn get_by_name(&self, name: &str) -> Option<DatasetManifest> {
let datasets = self.datasets.read().await;
datasets.values().find(|d| d.name == name).cloned()
}
/// List all datasets.
pub async fn list(&self) -> Vec<DatasetManifest> {
let datasets = self.datasets.read().await;
datasets.values().cloned().collect()
}
/// Add objects to an existing dataset.
pub async fn add_objects(
&self,
id: &DatasetId,
new_objects: Vec<ObjectRef>,
) -> Result<DatasetManifest, String> {
let mut datasets = self.datasets.write().await;
let manifest = datasets.get_mut(id).ok_or_else(|| format!("dataset not found: {id}"))?;
manifest.objects.extend(new_objects);
manifest.updated_at = chrono::Utc::now();
// Persist updated manifest
let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
let json = serde_json::to_vec_pretty(manifest).map_err(|e| e.to_string())?;
ops::put(&self.store, &manifest_key, json.into()).await?;
Ok(manifest.clone())
}
}