Phase 1: storage + catalog layer
- storaged: object_store backend (LocalFileSystem), PUT/GET/DELETE/LIST endpoints
- shared: arrow_helpers with Parquet roundtrip + schema fingerprinting (2 tests)
- catalogd: in-memory registry with write-ahead manifest persistence to object storage
- catalogd: POST/GET /datasets, GET /datasets/by-name/{name}
- gateway: wires storaged + catalogd with shared object_store state
- Phase tracker updated: Phase 0 + Phase 1 gates passed
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
use shared::types::{DatasetId, DatasetManifest, ObjectRef, SchemaFingerprint};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use storaged::ops;
|
||||
use object_store::ObjectStore;
|
||||
|
||||
const MANIFEST_PREFIX: &str = "_catalog/manifests";
|
||||
|
||||
/// In-memory dataset registry backed by manifest persistence in object storage.
|
||||
#[derive(Clone)]
|
||||
pub struct Registry {
|
||||
datasets: Arc<RwLock<HashMap<DatasetId, DatasetManifest>>>,
|
||||
store: Arc<dyn ObjectStore>,
|
||||
}
|
||||
|
||||
impl Registry {
|
||||
pub fn new(store: Arc<dyn ObjectStore>) -> Self {
|
||||
Self {
|
||||
datasets: Arc::new(RwLock::new(HashMap::new())),
|
||||
store,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild in-memory index from persisted manifests on startup.
|
||||
pub async fn rebuild(&self) -> Result<usize, String> {
|
||||
let keys = ops::list(&self.store, Some(MANIFEST_PREFIX)).await?;
|
||||
let mut datasets = self.datasets.write().await;
|
||||
datasets.clear();
|
||||
for key in &keys {
|
||||
let data = ops::get(&self.store, key).await?;
|
||||
let manifest: DatasetManifest =
|
||||
serde_json::from_slice(&data).map_err(|e| e.to_string())?;
|
||||
datasets.insert(manifest.id.clone(), manifest);
|
||||
}
|
||||
let count = datasets.len();
|
||||
tracing::info!("catalog rebuilt: {count} datasets loaded");
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Register a new dataset. Persists manifest to storage before updating memory.
|
||||
pub async fn register(
|
||||
&self,
|
||||
name: String,
|
||||
schema_fingerprint: SchemaFingerprint,
|
||||
objects: Vec<ObjectRef>,
|
||||
) -> Result<DatasetManifest, String> {
|
||||
let now = chrono::Utc::now();
|
||||
let manifest = DatasetManifest {
|
||||
id: DatasetId::new(),
|
||||
name,
|
||||
schema_fingerprint,
|
||||
objects,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
||||
// Write-ahead: persist before in-memory update
|
||||
let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
|
||||
let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?;
|
||||
ops::put(&self.store, &manifest_key, json.into()).await?;
|
||||
|
||||
let mut datasets = self.datasets.write().await;
|
||||
datasets.insert(manifest.id.clone(), manifest.clone());
|
||||
|
||||
tracing::info!("registered dataset: {} ({})", manifest.name, manifest.id);
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
/// Get a dataset by ID.
|
||||
pub async fn get(&self, id: &DatasetId) -> Option<DatasetManifest> {
|
||||
let datasets = self.datasets.read().await;
|
||||
datasets.get(id).cloned()
|
||||
}
|
||||
|
||||
/// Get a dataset by name.
|
||||
pub async fn get_by_name(&self, name: &str) -> Option<DatasetManifest> {
|
||||
let datasets = self.datasets.read().await;
|
||||
datasets.values().find(|d| d.name == name).cloned()
|
||||
}
|
||||
|
||||
/// List all datasets.
|
||||
pub async fn list(&self) -> Vec<DatasetManifest> {
|
||||
let datasets = self.datasets.read().await;
|
||||
datasets.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// Add objects to an existing dataset.
|
||||
pub async fn add_objects(
|
||||
&self,
|
||||
id: &DatasetId,
|
||||
new_objects: Vec<ObjectRef>,
|
||||
) -> Result<DatasetManifest, String> {
|
||||
let mut datasets = self.datasets.write().await;
|
||||
let manifest = datasets.get_mut(id).ok_or_else(|| format!("dataset not found: {id}"))?;
|
||||
|
||||
manifest.objects.extend(new_objects);
|
||||
manifest.updated_at = chrono::Utc::now();
|
||||
|
||||
// Persist updated manifest
|
||||
let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
|
||||
let json = serde_json::to_vec_pretty(manifest).map_err(|e| e.to_string())?;
|
||||
ops::put(&self.store, &manifest_key, json.into()).await?;
|
||||
|
||||
Ok(manifest.clone())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user