Phase 1: storage + catalog layer
- storaged: object_store backend (LocalFileSystem), PUT/GET/DELETE/LIST endpoints
- shared: arrow_helpers with Parquet roundtrip + schema fingerprinting (2 tests)
- catalogd: in-memory registry with write-ahead manifest persistence to object storage
- catalogd: POST/GET /datasets, GET /datasets/by-name/{name}
- gateway: wires storaged + catalogd with shared object_store state
- Phase tracker updated: Phase 0 + Phase 1 gates passed
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,9 +1,125 @@
|
||||
use axum::{Router, routing::get};
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use shared::types::{DatasetId, ObjectRef, SchemaFingerprint};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn router() -> Router {
|
||||
Router::new().route("/health", get(health))
|
||||
use crate::registry::Registry;
|
||||
|
||||
pub fn router(registry: Registry) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/datasets", post(create_dataset))
|
||||
.route("/datasets", get(list_datasets))
|
||||
.route("/datasets/{id}", get(get_dataset))
|
||||
.route("/datasets/by-name/{name}", get(get_dataset_by_name))
|
||||
.with_state(registry)
|
||||
}
|
||||
|
||||
async fn health() -> &'static str {
|
||||
"catalogd ok"
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateDatasetRequest {
|
||||
name: String,
|
||||
schema_fingerprint: String,
|
||||
objects: Vec<ObjectRefRequest>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ObjectRefRequest {
|
||||
bucket: String,
|
||||
key: String,
|
||||
size_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct DatasetResponse {
|
||||
id: String,
|
||||
name: String,
|
||||
schema_fingerprint: String,
|
||||
objects: Vec<ObjectRefResponse>,
|
||||
created_at: String,
|
||||
updated_at: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ObjectRefResponse {
|
||||
bucket: String,
|
||||
key: String,
|
||||
size_bytes: u64,
|
||||
created_at: String,
|
||||
}
|
||||
|
||||
impl From<&shared::types::DatasetManifest> for DatasetResponse {
|
||||
fn from(m: &shared::types::DatasetManifest) -> Self {
|
||||
Self {
|
||||
id: m.id.to_string(),
|
||||
name: m.name.clone(),
|
||||
schema_fingerprint: m.schema_fingerprint.0.clone(),
|
||||
objects: m.objects.iter().map(|o| ObjectRefResponse {
|
||||
bucket: o.bucket.clone(),
|
||||
key: o.key.clone(),
|
||||
size_bytes: o.size_bytes,
|
||||
created_at: o.created_at.to_rfc3339(),
|
||||
}).collect(),
|
||||
created_at: m.created_at.to_rfc3339(),
|
||||
updated_at: m.updated_at.to_rfc3339(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_dataset(
|
||||
State(registry): State<Registry>,
|
||||
Json(req): Json<CreateDatasetRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let now = chrono::Utc::now();
|
||||
let objects: Vec<ObjectRef> = req.objects.into_iter().map(|o| ObjectRef {
|
||||
bucket: o.bucket,
|
||||
key: o.key,
|
||||
size_bytes: o.size_bytes,
|
||||
created_at: now,
|
||||
}).collect();
|
||||
|
||||
match registry.register(req.name, SchemaFingerprint(req.schema_fingerprint), objects).await {
|
||||
Ok(manifest) => {
|
||||
let resp = DatasetResponse::from(&manifest);
|
||||
Ok((StatusCode::CREATED, Json(resp)))
|
||||
}
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_datasets(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
let datasets = registry.list().await;
|
||||
let resp: Vec<DatasetResponse> = datasets.iter().map(DatasetResponse::from).collect();
|
||||
Json(resp)
|
||||
}
|
||||
|
||||
async fn get_dataset(
|
||||
State(registry): State<Registry>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
let uuid = Uuid::parse_str(&id).map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
|
||||
let dataset_id = DatasetId(uuid);
|
||||
match registry.get(&dataset_id).await {
|
||||
Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {id}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_dataset_by_name(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.get_by_name(&name).await {
|
||||
Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {name}"))),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user