Phase 1: storage + catalog layer

- storaged: object_store backend (LocalFileSystem), PUT/GET/DELETE/LIST endpoints
- shared: arrow_helpers with Parquet roundtrip + schema fingerprinting (2 tests)
- catalogd: in-memory registry with write-ahead manifest persistence to object storage
- catalogd: POST/GET /datasets, GET /datasets/by-name/{name}
- gateway: wires storaged + catalogd with shared object_store state
- Phase tracker updated: Phase 0 + Phase 1 gates passed

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 05:15:27 -05:00
co-authored by Claude Opus 4.6
parent a52ca841c6
commit 655b6c0b37
17 changed files with 2331 additions and 30 deletions
+119 -3
View File
@@ -1,9 +1,125 @@
use axum::{Router, routing::get};
use axum::{
Json, Router,
extract::{Path, State},
http::StatusCode,
response::IntoResponse,
routing::{get, post},
};
use serde::{Deserialize, Serialize};
use shared::types::{DatasetId, ObjectRef, SchemaFingerprint};
use uuid::Uuid;
pub fn router() -> Router {
Router::new().route("/health", get(health))
use crate::registry::Registry;
pub fn router(registry: Registry) -> Router {
Router::new()
.route("/health", get(health))
.route("/datasets", post(create_dataset))
.route("/datasets", get(list_datasets))
.route("/datasets/{id}", get(get_dataset))
.route("/datasets/by-name/{name}", get(get_dataset_by_name))
.with_state(registry)
}
async fn health() -> &'static str {
"catalogd ok"
}
#[derive(Deserialize)]
struct CreateDatasetRequest {
name: String,
schema_fingerprint: String,
objects: Vec<ObjectRefRequest>,
}
#[derive(Deserialize)]
struct ObjectRefRequest {
bucket: String,
key: String,
size_bytes: u64,
}
#[derive(Serialize)]
struct DatasetResponse {
id: String,
name: String,
schema_fingerprint: String,
objects: Vec<ObjectRefResponse>,
created_at: String,
updated_at: String,
}
#[derive(Serialize)]
struct ObjectRefResponse {
bucket: String,
key: String,
size_bytes: u64,
created_at: String,
}
impl From<&shared::types::DatasetManifest> for DatasetResponse {
fn from(m: &shared::types::DatasetManifest) -> Self {
Self {
id: m.id.to_string(),
name: m.name.clone(),
schema_fingerprint: m.schema_fingerprint.0.clone(),
objects: m.objects.iter().map(|o| ObjectRefResponse {
bucket: o.bucket.clone(),
key: o.key.clone(),
size_bytes: o.size_bytes,
created_at: o.created_at.to_rfc3339(),
}).collect(),
created_at: m.created_at.to_rfc3339(),
updated_at: m.updated_at.to_rfc3339(),
}
}
}
async fn create_dataset(
State(registry): State<Registry>,
Json(req): Json<CreateDatasetRequest>,
) -> impl IntoResponse {
let now = chrono::Utc::now();
let objects: Vec<ObjectRef> = req.objects.into_iter().map(|o| ObjectRef {
bucket: o.bucket,
key: o.key,
size_bytes: o.size_bytes,
created_at: now,
}).collect();
match registry.register(req.name, SchemaFingerprint(req.schema_fingerprint), objects).await {
Ok(manifest) => {
let resp = DatasetResponse::from(&manifest);
Ok((StatusCode::CREATED, Json(resp)))
}
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
}
}
async fn list_datasets(State(registry): State<Registry>) -> impl IntoResponse {
let datasets = registry.list().await;
let resp: Vec<DatasetResponse> = datasets.iter().map(DatasetResponse::from).collect();
Json(resp)
}
async fn get_dataset(
State(registry): State<Registry>,
Path(id): Path<String>,
) -> impl IntoResponse {
let uuid = Uuid::parse_str(&id).map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
let dataset_id = DatasetId(uuid);
match registry.get(&dataset_id).await {
Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {id}"))),
}
}
async fn get_dataset_by_name(
State(registry): State<Registry>,
Path(name): Path<String>,
) -> impl IntoResponse {
match registry.get_by_name(&name).await {
Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {name}"))),
}
}