checkpoint: matrix-agent-validated (2026-04-25)
Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.
WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.
WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
* UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
* REVISE: chains versions, parent.superseded_at + superseded_by stamped
* RETIRE: marks specific trace retired with reason, excluded from retrieval
* HISTORY: walks chain root→tip, cycle-safe
KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces
Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
profit
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "storaged"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
object_store = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
@@ -0,0 +1,330 @@
|
||||
/// Write-once batched append log.
|
||||
///
|
||||
/// Problem we're fixing: the error journal and HNSW trial journal both
|
||||
/// previously did read-modify-write of their whole JSONL file on every
|
||||
/// event. That's O(N²) cumulative work and generates huge churn at scale.
|
||||
/// It's exactly the pattern llms3.com flags as the "small-file /
|
||||
/// rewrite-amplification" anti-pattern.
|
||||
///
|
||||
/// This helper implements the pattern object storage actually wants:
|
||||
///
|
||||
/// - Events accumulate in an **in-memory buffer** (reads see them immediately).
|
||||
/// - When the buffer hits a threshold, or `flush()` is called, the buffer is
|
||||
/// written **as one new object** with a timestamp-sorted key.
|
||||
/// - Existing objects are never rewritten.
|
||||
/// - Reads enumerate all batch files, sort by key, and concat in order.
|
||||
/// - An explicit `compact()` reads every batch file, writes one consolidated
|
||||
/// file, and deletes the originals — the LSM-tree compaction idea applied
|
||||
/// to small JSONL events.
|
||||
///
|
||||
/// Storage layout:
|
||||
/// ```text
|
||||
/// {prefix}/
|
||||
/// batch_0001776319628000123.jsonl
|
||||
/// batch_0001776319745987654.jsonl
|
||||
/// batch_compacted_00001776319800000000.jsonl (after compact())
|
||||
/// ```
|
||||
/// Key format: the zero-padded epoch microsecond of the write, so lexical
|
||||
/// sort == chronological sort.
|
||||
|
||||
use bytes::Bytes;
|
||||
use chrono::Utc;
|
||||
use object_store::ObjectStore;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::ops;
|
||||
|
||||
const DEFAULT_FLUSH_THRESHOLD: usize = 32;
|
||||
|
||||
pub struct AppendLog {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
prefix: String,
|
||||
buffer: Mutex<Vec<Vec<u8>>>,
|
||||
flush_threshold: usize,
|
||||
}
|
||||
|
||||
impl AppendLog {
|
||||
/// Create a new append log rooted at `prefix` in the given object store.
|
||||
/// Events auto-flush when buffer reaches `flush_threshold` (default 32).
|
||||
pub fn new(store: Arc<dyn ObjectStore>, prefix: impl Into<String>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
prefix: prefix.into(),
|
||||
buffer: Mutex::new(Vec::new()),
|
||||
flush_threshold: DEFAULT_FLUSH_THRESHOLD,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_flush_threshold(mut self, threshold: usize) -> Self {
|
||||
self.flush_threshold = threshold.max(1);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add an event. The returned future completes either immediately
|
||||
/// (buffered) or after a flush, depending on whether the buffer hit the
|
||||
/// threshold. Callers don't need to care either way.
|
||||
pub async fn append(&self, line: Vec<u8>) -> Result<(), String> {
|
||||
let should_flush = {
|
||||
let mut buf = self.buffer.lock().await;
|
||||
buf.push(line);
|
||||
buf.len() >= self.flush_threshold
|
||||
};
|
||||
if should_flush {
|
||||
self.flush().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Force-flush the in-memory buffer to object storage as a single new
|
||||
/// batch file. Safe to call anytime (idempotent no-op when buffer empty).
|
||||
pub async fn flush(&self) -> Result<(), String> {
|
||||
let batch = {
|
||||
let mut buf = self.buffer.lock().await;
|
||||
if buf.is_empty() { return Ok(()); }
|
||||
std::mem::take(&mut *buf)
|
||||
};
|
||||
|
||||
let ts_us = Utc::now().timestamp_micros().max(0) as u128;
|
||||
let key = format!("{}/batch_{:019}.jsonl", self.prefix, ts_us);
|
||||
|
||||
let mut body = Vec::with_capacity(batch.iter().map(|b| b.len() + 1).sum());
|
||||
for line in batch {
|
||||
body.extend_from_slice(&line);
|
||||
if !body.ends_with(b"\n") { body.push(b'\n'); }
|
||||
}
|
||||
ops::put(&self.store, &key, Bytes::from(body)).await
|
||||
}
|
||||
|
||||
/// Read every event across all batch files + unflushed in-memory buffer.
|
||||
/// Events are returned in chronological order.
|
||||
pub async fn read_all(&self) -> Result<Vec<Vec<u8>>, String> {
|
||||
let mut keys = self.list_batch_keys().await?;
|
||||
keys.sort();
|
||||
|
||||
let mut out = Vec::new();
|
||||
for key in keys {
|
||||
let bytes = ops::get(&self.store, &key).await?;
|
||||
for line in bytes.split(|b| *b == b'\n') {
|
||||
if !line.is_empty() {
|
||||
out.push(line.to_vec());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Include unflushed events so callers see the latest state
|
||||
// whether or not someone ran flush() recently.
|
||||
let buf = self.buffer.lock().await;
|
||||
for line in buf.iter() {
|
||||
out.push(line.clone());
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Consolidate all current batch files into one compacted file, then
|
||||
/// delete the originals. Safe to call while appends are in flight:
|
||||
/// new batches written during compaction get a higher timestamp and
|
||||
/// survive. Fails closed — if anything goes wrong mid-delete, the
|
||||
/// compacted file coexists with originals and next read sees duplicates
|
||||
/// (which the dedup caller must handle) rather than data loss.
|
||||
pub async fn compact(&self) -> Result<CompactStats, String> {
|
||||
// Snapshot which files to compact BEFORE we write the new one.
|
||||
let mut originals = self.list_batch_keys().await?;
|
||||
originals.sort();
|
||||
|
||||
if originals.len() < 2 {
|
||||
return Ok(CompactStats { merged_files: originals.len(), events: 0, new_key: None });
|
||||
}
|
||||
|
||||
// Gather all existing events.
|
||||
let mut events = Vec::new();
|
||||
for key in &originals {
|
||||
let bytes = ops::get(&self.store, key).await?;
|
||||
for line in bytes.split(|b| *b == b'\n') {
|
||||
if !line.is_empty() {
|
||||
events.push(line.to_vec());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_events = events.len();
|
||||
if total_events == 0 {
|
||||
// Clean up empty files without writing a new one.
|
||||
for key in &originals {
|
||||
let _ = ops::delete(&self.store, key).await;
|
||||
}
|
||||
return Ok(CompactStats { merged_files: originals.len(), events: 0, new_key: None });
|
||||
}
|
||||
|
||||
// Name the compacted file with the SAME `batch_{ts}` format so it
|
||||
// sorts chronologically with future batches. Using a distinct prefix
|
||||
// ("batch_compacted_") would break lex ordering: later `batch_N`
|
||||
// files would sort BEFORE the compacted file because 'c' > digits.
|
||||
// Timestamp = now, so any appends arriving during compaction (which
|
||||
// get the current wall-clock time) sort AFTER this file.
|
||||
let ts_us = Utc::now().timestamp_micros().max(0) as u128;
|
||||
let new_key = format!("{}/batch_{:019}.jsonl", self.prefix, ts_us);
|
||||
|
||||
let mut body = Vec::new();
|
||||
for line in &events {
|
||||
body.extend_from_slice(line);
|
||||
body.push(b'\n');
|
||||
}
|
||||
ops::put(&self.store, &new_key, Bytes::from(body)).await?;
|
||||
|
||||
// Only delete originals once the consolidated file is persisted.
|
||||
let mut failures = 0;
|
||||
for key in &originals {
|
||||
if ops::delete(&self.store, key).await.is_err() {
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
if failures > 0 {
|
||||
tracing::warn!(
|
||||
"compact '{}': {} original files failed to delete — consolidated file {} has the data",
|
||||
self.prefix, failures, new_key,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(CompactStats {
|
||||
merged_files: originals.len(),
|
||||
events: total_events,
|
||||
new_key: Some(new_key),
|
||||
})
|
||||
}
|
||||
|
||||
/// How many batch files exist on disk right now.
|
||||
pub async fn file_count(&self) -> Result<usize, String> {
|
||||
Ok(self.list_batch_keys().await?.len())
|
||||
}
|
||||
|
||||
async fn list_batch_keys(&self) -> Result<Vec<String>, String> {
|
||||
let prefix_with_slash = format!("{}/", self.prefix);
|
||||
// list the prefix then filter for keys that match our naming scheme;
|
||||
// unrelated files at the same prefix won't be touched.
|
||||
let raw = ops::list(&self.store, Some(&prefix_with_slash)).await?;
|
||||
Ok(raw
|
||||
.into_iter()
|
||||
.filter(|k| {
|
||||
let basename = k.rsplit('/').next().unwrap_or(k);
|
||||
basename.starts_with("batch_") && basename.ends_with(".jsonl")
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct CompactStats {
|
||||
pub merged_files: usize,
|
||||
pub events: usize,
|
||||
pub new_key: Option<String>,
|
||||
}
|
||||
|
||||
// Log unflushed-buffer size on drop. We can't `.await` from a sync `Drop`,
|
||||
// so a real flush isn't possible here — callers are responsible for calling
|
||||
// `.flush()` before dropping if durability matters. These journals are
|
||||
// observability hints; a few lost buffered events at shutdown are
|
||||
// acceptable per ADR-018.
|
||||
impl Drop for AppendLog {
|
||||
fn drop(&mut self) {
|
||||
let buf_len = self.buffer.try_lock().map(|b| b.len()).unwrap_or(0);
|
||||
if buf_len == 0 { return; }
|
||||
tracing::debug!(
|
||||
"append_log '{}' dropping with {} unflushed events",
|
||||
self.prefix, buf_len,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use object_store::memory::InMemory;
|
||||
|
||||
fn mk(threshold: usize) -> AppendLog {
|
||||
AppendLog::new(Arc::new(InMemory::new()), "prefix")
|
||||
.with_flush_threshold(threshold)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn append_stays_buffered_below_threshold() {
|
||||
let log = mk(5);
|
||||
log.append(b"one".to_vec()).await.unwrap();
|
||||
log.append(b"two".to_vec()).await.unwrap();
|
||||
assert_eq!(log.file_count().await.unwrap(), 0, "no files until threshold");
|
||||
let all = log.read_all().await.unwrap();
|
||||
assert_eq!(all, vec![b"one".to_vec(), b"two".to_vec()],
|
||||
"read_all surfaces unflushed buffer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn append_auto_flushes_on_threshold() {
|
||||
let log = mk(3);
|
||||
for i in 0..3 {
|
||||
log.append(format!("evt{i}").into_bytes()).await.unwrap();
|
||||
}
|
||||
assert_eq!(log.file_count().await.unwrap(), 1, "threshold triggered one flush");
|
||||
|
||||
// A fourth append stays buffered until the next threshold.
|
||||
log.append(b"evt3".to_vec()).await.unwrap();
|
||||
assert_eq!(log.file_count().await.unwrap(), 1, "below threshold again");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn flush_empty_is_noop() {
|
||||
let log = mk(32);
|
||||
log.flush().await.unwrap();
|
||||
log.flush().await.unwrap();
|
||||
assert_eq!(log.file_count().await.unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_all_orders_events_across_flushes() {
|
||||
let log = mk(1); // flush-on-every-append
|
||||
for i in 0..5 {
|
||||
log.append(format!("e{i}").into_bytes()).await.unwrap();
|
||||
// Spread writes out so timestamps sort strictly.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||
}
|
||||
let all = log.read_all().await.unwrap();
|
||||
let strs: Vec<String> = all.into_iter()
|
||||
.map(|v| String::from_utf8(v).unwrap())
|
||||
.collect();
|
||||
assert_eq!(strs, vec!["e0", "e1", "e2", "e3", "e4"],
|
||||
"lex sort of batch keys == chronological event order");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_merges_multiple_files_into_one() {
|
||||
let log = mk(1); // force file-per-append
|
||||
for i in 0..4 {
|
||||
log.append(format!("e{i}").into_bytes()).await.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||
}
|
||||
assert_eq!(log.file_count().await.unwrap(), 4);
|
||||
|
||||
let stats = log.compact().await.unwrap();
|
||||
assert_eq!(stats.merged_files, 4);
|
||||
assert_eq!(stats.events, 4);
|
||||
assert!(stats.new_key.is_some());
|
||||
|
||||
assert_eq!(log.file_count().await.unwrap(), 1, "originals deleted, 1 survivor");
|
||||
let all = log.read_all().await.unwrap();
|
||||
assert_eq!(all.len(), 4, "no events lost in compaction");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_with_single_file_is_noop() {
|
||||
let log = mk(1);
|
||||
log.append(b"only".to_vec()).await.unwrap();
|
||||
assert_eq!(log.file_count().await.unwrap(), 1);
|
||||
|
||||
let stats = log.compact().await.unwrap();
|
||||
assert_eq!(stats.merged_files, 1);
|
||||
assert_eq!(stats.events, 0, "nothing to consolidate");
|
||||
assert!(stats.new_key.is_none(), "no new file written");
|
||||
assert_eq!(log.file_count().await.unwrap(), 1, "original untouched");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
use object_store::ObjectStore;
|
||||
use object_store::local::LocalFileSystem;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Initialize the object store backend.
|
||||
/// Starts with LocalFileSystem. Swap to S3/RustFS by changing this function.
|
||||
pub fn init_local(root: &str) -> Arc<dyn ObjectStore> {
|
||||
std::fs::create_dir_all(root).expect("failed to create storage root");
|
||||
Arc::new(LocalFileSystem::new_with_prefix(root).expect("failed to init local object store"))
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
/// Bucket operation error journal.
|
||||
///
|
||||
/// Every bucket op failure and every rescue fallback lands here. Goal:
|
||||
/// answering "is anything broken?" with one HTTP call.
|
||||
///
|
||||
/// Storage: batched write-once files under `_errors/bucket_errors/` in the
|
||||
/// primary bucket. Uses the shared `AppendLog` helper so we never rewrite
|
||||
/// existing files — see `append_log.rs` for the full pattern rationale.
|
||||
///
|
||||
/// In-memory ring buffer holds the last N events for fast response; on
|
||||
/// startup `load_recent` hydrates it from all batch files.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use object_store::ObjectStore;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::append_log::{AppendLog, CompactStats};
|
||||
|
||||
const JOURNAL_PREFIX: &str = "_errors/bucket_errors";
|
||||
const RING_CAPACITY: usize = 2000;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum BucketOp {
|
||||
Read,
|
||||
Write,
|
||||
Delete,
|
||||
List,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BucketErrorEvent {
|
||||
pub ts: DateTime<Utc>,
|
||||
pub op: BucketOp,
|
||||
pub target: String,
|
||||
pub key: String,
|
||||
pub error: String,
|
||||
#[serde(default)]
|
||||
pub rescued: bool,
|
||||
}
|
||||
|
||||
impl BucketErrorEvent {
|
||||
pub fn new_read(target: &str, key: &str, error: &str) -> Self {
|
||||
Self { ts: Utc::now(), op: BucketOp::Read, target: target.into(), key: key.into(), error: error.into(), rescued: false }
|
||||
}
|
||||
pub fn new_write(target: &str, key: &str, error: &str) -> Self {
|
||||
Self { ts: Utc::now(), op: BucketOp::Write, target: target.into(), key: key.into(), error: error.into(), rescued: false }
|
||||
}
|
||||
pub fn new_delete(target: &str, key: &str, error: &str) -> Self {
|
||||
Self { ts: Utc::now(), op: BucketOp::Delete, target: target.into(), key: key.into(), error: error.into(), rescued: false }
|
||||
}
|
||||
pub fn new_list(target: &str, prefix: &str, error: &str) -> Self {
|
||||
Self { ts: Utc::now(), op: BucketOp::List, target: target.into(), key: prefix.into(), error: error.into(), rescued: false }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ErrorJournal {
|
||||
log: Arc<AppendLog>,
|
||||
ring: Arc<RwLock<VecDeque<BucketErrorEvent>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HealthReport {
|
||||
pub period_minutes: i64,
|
||||
pub total_errors: usize,
|
||||
pub per_bucket: std::collections::HashMap<String, usize>,
|
||||
pub unhealthy_buckets: Vec<String>,
|
||||
}
|
||||
|
||||
impl ErrorJournal {
|
||||
pub fn new(store: Arc<dyn ObjectStore>) -> Self {
|
||||
// Keep flush threshold lowish — operators checking /storage/errors
|
||||
// after a recent incident want to see fresh rows even without an
|
||||
// explicit flush.
|
||||
let log = Arc::new(
|
||||
AppendLog::new(store, JOURNAL_PREFIX).with_flush_threshold(8),
|
||||
);
|
||||
Self {
|
||||
log,
|
||||
ring: Arc::new(RwLock::new(VecDeque::with_capacity(RING_CAPACITY))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Hydrate the ring buffer from existing batch files. Call once at
|
||||
/// startup. Tolerates malformed lines (skipped with a warning) and
|
||||
/// missing files (returns 0).
|
||||
pub async fn load_recent(&self) -> Result<usize, String> {
|
||||
let lines = self.log.read_all().await.unwrap_or_default();
|
||||
let mut ring = self.ring.write().await;
|
||||
for line in lines {
|
||||
match serde_json::from_slice::<BucketErrorEvent>(&line) {
|
||||
Ok(ev) => {
|
||||
if ring.len() >= RING_CAPACITY { ring.pop_front(); }
|
||||
ring.push_back(ev);
|
||||
}
|
||||
Err(e) => tracing::warn!("error journal: skip malformed line ({e})"),
|
||||
}
|
||||
}
|
||||
Ok(ring.len())
|
||||
}
|
||||
|
||||
/// Append an event. In-memory ring updated immediately; persistence
|
||||
/// happens in batches via the underlying AppendLog.
|
||||
pub async fn append(&self, event: BucketErrorEvent) {
|
||||
{
|
||||
let mut ring = self.ring.write().await;
|
||||
if ring.len() >= RING_CAPACITY { ring.pop_front(); }
|
||||
ring.push_back(event.clone());
|
||||
}
|
||||
|
||||
match serde_json::to_vec(&event) {
|
||||
Ok(line) => {
|
||||
if let Err(e) = self.log.append(line).await {
|
||||
tracing::error!("error journal persist failed: {e}");
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("error journal serialize failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark the most recent matching in-memory event as rescued.
|
||||
/// Only updates the ring buffer — the JSONL line already persisted
|
||||
/// records the failure fact; rescue status travels in response headers.
|
||||
pub async fn mark_rescued_last(&self, target: &str, key: &str) {
|
||||
let mut ring = self.ring.write().await;
|
||||
for ev in ring.iter_mut().rev() {
|
||||
if matches!(ev.op, BucketOp::Read) && ev.target == target && ev.key == key && !ev.rescued {
|
||||
ev.rescued = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn recent(&self, limit: usize) -> Vec<BucketErrorEvent> {
|
||||
let ring = self.ring.read().await;
|
||||
let start = ring.len().saturating_sub(limit);
|
||||
ring.iter().skip(start).cloned().collect()
|
||||
}
|
||||
|
||||
pub async fn filter(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
since: Option<DateTime<Utc>>,
|
||||
limit: usize,
|
||||
) -> Vec<BucketErrorEvent> {
|
||||
let ring = self.ring.read().await;
|
||||
ring.iter()
|
||||
.rev()
|
||||
.filter(|ev| bucket.map_or(true, |b| ev.target == b))
|
||||
.filter(|ev| since.map_or(true, |s| ev.ts >= s))
|
||||
.take(limit)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Summarize errors in the last N minutes.
|
||||
pub async fn health(&self, period_minutes: i64) -> HealthReport {
|
||||
use std::collections::HashMap;
|
||||
let cutoff = Utc::now() - chrono::Duration::minutes(period_minutes);
|
||||
let ring = self.ring.read().await;
|
||||
let recent: Vec<_> = ring.iter().filter(|ev| ev.ts >= cutoff).collect();
|
||||
let mut per_bucket: HashMap<String, usize> = HashMap::new();
|
||||
for ev in &recent {
|
||||
*per_bucket.entry(ev.target.clone()).or_insert(0) += 1;
|
||||
}
|
||||
let unhealthy_buckets: Vec<String> = per_bucket
|
||||
.iter()
|
||||
.filter(|(_, c)| **c >= 3)
|
||||
.map(|(k, _)| k.clone())
|
||||
.collect();
|
||||
HealthReport {
|
||||
period_minutes,
|
||||
total_errors: recent.len(),
|
||||
per_bucket,
|
||||
unhealthy_buckets,
|
||||
}
|
||||
}
|
||||
|
||||
/// Force an immediate flush of buffered events to object storage.
|
||||
pub async fn flush(&self) -> Result<(), String> {
|
||||
self.log.flush().await
|
||||
}
|
||||
|
||||
/// Consolidate all batch files into one. Operator cleanup.
|
||||
pub async fn compact(&self) -> Result<CompactStats, String> {
|
||||
self.log.compact().await
|
||||
}
|
||||
|
||||
/// How many JSONL batch files currently exist.
|
||||
pub async fn file_count(&self) -> Result<usize, String> {
|
||||
self.log.file_count().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use object_store::memory::InMemory;
|
||||
|
||||
fn mk() -> ErrorJournal {
|
||||
ErrorJournal::new(Arc::new(InMemory::new()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn append_updates_ring_immediately() {
|
||||
let j = mk();
|
||||
j.append(BucketErrorEvent::new_read("tenant_a", "k1", "timeout")).await;
|
||||
j.append(BucketErrorEvent::new_write("tenant_b", "k2", "denied")).await;
|
||||
|
||||
let recent = j.recent(10).await;
|
||||
assert_eq!(recent.len(), 2);
|
||||
assert_eq!(recent[0].target, "tenant_a");
|
||||
assert_eq!(recent[1].target, "tenant_b");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_recent_rehydrates_ring_from_disk() {
|
||||
// Use a shared InMemory store so the second journal reads the first
|
||||
// journal's writes — mirrors gateway restart semantics.
|
||||
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
|
||||
{
|
||||
let j1 = ErrorJournal {
|
||||
log: Arc::new(AppendLog::new(store.clone(), JOURNAL_PREFIX).with_flush_threshold(1)),
|
||||
ring: Arc::new(RwLock::new(VecDeque::with_capacity(RING_CAPACITY))),
|
||||
};
|
||||
j1.append(BucketErrorEvent::new_read("t", "k", "err")).await;
|
||||
j1.flush().await.unwrap();
|
||||
}
|
||||
|
||||
let j2 = ErrorJournal {
|
||||
log: Arc::new(AppendLog::new(store.clone(), JOURNAL_PREFIX)),
|
||||
ring: Arc::new(RwLock::new(VecDeque::with_capacity(RING_CAPACITY))),
|
||||
};
|
||||
assert_eq!(j2.recent(10).await.len(), 0, "fresh ring is empty");
|
||||
|
||||
let loaded = j2.load_recent().await.unwrap();
|
||||
assert_eq!(loaded, 1);
|
||||
assert_eq!(j2.recent(10).await[0].target, "t");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mark_rescued_updates_most_recent_match() {
|
||||
let j = mk();
|
||||
j.append(BucketErrorEvent::new_read("t1", "k", "err")).await;
|
||||
j.append(BucketErrorEvent::new_read("t2", "k", "err")).await;
|
||||
j.append(BucketErrorEvent::new_read("t1", "k", "err")).await;
|
||||
|
||||
j.mark_rescued_last("t1", "k").await;
|
||||
|
||||
let recent = j.recent(10).await;
|
||||
assert!(!recent[0].rescued, "older t1 event not touched");
|
||||
assert!(!recent[1].rescued, "t2 not touched");
|
||||
assert!(recent[2].rescued, "most recent t1 was rescued");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_flags_buckets_with_three_or_more_errors() {
|
||||
let j = mk();
|
||||
for _ in 0..3 { j.append(BucketErrorEvent::new_read("broken", "k", "e")).await; }
|
||||
j.append(BucketErrorEvent::new_read("flaky", "k", "e")).await;
|
||||
|
||||
let h = j.health(60).await;
|
||||
assert_eq!(h.total_errors, 4);
|
||||
assert_eq!(h.per_bucket.get("broken"), Some(&3));
|
||||
assert_eq!(h.per_bucket.get("flaky"), Some(&1));
|
||||
assert!(h.unhealthy_buckets.contains(&"broken".to_string()));
|
||||
assert!(!h.unhealthy_buckets.contains(&"flaky".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn filter_by_bucket_returns_chronological_order() {
|
||||
let j = mk();
|
||||
j.append(BucketErrorEvent::new_read("a", "k1", "e")).await;
|
||||
j.append(BucketErrorEvent::new_read("b", "k2", "e")).await;
|
||||
j.append(BucketErrorEvent::new_read("a", "k3", "e")).await;
|
||||
|
||||
let only_a = j.filter(Some("a"), None, 10).await;
|
||||
assert_eq!(only_a.len(), 2);
|
||||
assert_eq!(only_a[0].key, "k1", "oldest first");
|
||||
assert_eq!(only_a[1].key, "k3");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
/// HTTP surface for federation operator endpoints.
|
||||
///
|
||||
/// - `GET /buckets` — configured buckets with reachability status
|
||||
/// - `GET /errors` — recent bucket op failures (filterable)
|
||||
/// - `GET /bucket-health` — aggregated errors-per-bucket in the last 5 minutes
|
||||
///
|
||||
/// Mounted under `/storage` by the gateway. `/storage/health` is already
|
||||
/// claimed by the existing storage router for service liveness, so the
|
||||
/// aggregated bucket health lives at `/storage/bucket-health`.
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{delete, get, post, put},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Deserialize;
|
||||
use shared::config::BucketConfig;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::error_journal::BucketErrorEvent;
|
||||
use crate::registry::{BucketInfo, BucketRegistry};
|
||||
|
||||
pub fn router(registry: Arc<BucketRegistry>) -> Router {
|
||||
Router::new()
|
||||
.route("/buckets", get(list_buckets).post(add_bucket))
|
||||
.route("/buckets/{name}", delete(remove_bucket))
|
||||
.route("/errors", get(list_errors))
|
||||
.route("/errors/flush", post(flush_errors))
|
||||
.route("/errors/compact", post(compact_errors))
|
||||
.route("/bucket-health", get(get_health))
|
||||
.route("/buckets/{bucket}/objects/{*key}", put(put_bucket_object))
|
||||
.route("/buckets/{bucket}/objects/{*key}", get(get_bucket_object))
|
||||
.with_state(registry)
|
||||
}
|
||||
|
||||
/// Provision + register a bucket at runtime. Body is a `BucketConfig`.
|
||||
///
|
||||
/// Federation layer 2: profile buckets (`profile:alice`) and tenant
|
||||
/// buckets can be added after startup without a service restart.
|
||||
async fn add_bucket(
|
||||
State(reg): State<Arc<BucketRegistry>>,
|
||||
Json(bc): Json<BucketConfig>,
|
||||
) -> impl IntoResponse {
|
||||
// Safety net: local backends must land somewhere under the configured
|
||||
// profile_root (for profile:*) or be otherwise explicitly rooted.
|
||||
// Refuse empty / missing root to avoid "bucket created at ./" surprises.
|
||||
if bc.backend == "local" {
|
||||
match bc.root.as_deref() {
|
||||
Some(r) if !r.trim().is_empty() => {}
|
||||
_ => return Err((StatusCode::BAD_REQUEST,
|
||||
"local bucket requires a non-empty 'root' path".to_string())),
|
||||
}
|
||||
}
|
||||
match reg.add_bucket(bc).await {
|
||||
Ok(info) => Ok((StatusCode::CREATED, Json(info))),
|
||||
Err(e) => {
|
||||
let code = if e.contains("already registered") {
|
||||
StatusCode::CONFLICT
|
||||
} else {
|
||||
StatusCode::BAD_REQUEST
|
||||
};
|
||||
Err((code, e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Unregister a bucket. Refused for primary / rescue / unknown names.
|
||||
async fn remove_bucket(
|
||||
State(reg): State<Arc<BucketRegistry>>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match reg.remove_bucket(&name) {
|
||||
Ok(()) => Ok((StatusCode::OK, format!("unregistered: {name}"))),
|
||||
Err(e) => {
|
||||
let code = if e.contains("cannot remove") {
|
||||
StatusCode::FORBIDDEN
|
||||
} else if e.contains("not registered") {
|
||||
StatusCode::NOT_FOUND
|
||||
} else {
|
||||
StatusCode::BAD_REQUEST
|
||||
};
|
||||
Err((code, e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_buckets(State(reg): State<Arc<BucketRegistry>>) -> Json<Vec<BucketInfo>> {
|
||||
Json(reg.list().await)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ErrorQuery {
|
||||
bucket: Option<String>,
|
||||
since: Option<DateTime<Utc>>,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: usize,
|
||||
}
|
||||
|
||||
fn default_limit() -> usize { 50 }
|
||||
|
||||
async fn list_errors(
|
||||
State(reg): State<Arc<BucketRegistry>>,
|
||||
Query(q): Query<ErrorQuery>,
|
||||
) -> Json<Vec<BucketErrorEvent>> {
|
||||
Json(reg.journal().filter(q.bucket.as_deref(), q.since, q.limit).await)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct HealthQuery {
|
||||
#[serde(default = "default_period")]
|
||||
minutes: i64,
|
||||
}
|
||||
|
||||
fn default_period() -> i64 { 5 }
|
||||
|
||||
async fn get_health(
|
||||
State(reg): State<Arc<BucketRegistry>>,
|
||||
Query(q): Query<HealthQuery>,
|
||||
) -> impl IntoResponse {
|
||||
Json(reg.journal().health(q.minutes).await)
|
||||
}
|
||||
|
||||
/// Bucket-aware write. Hard-fails if the target bucket is unreachable;
|
||||
/// never falls back to rescue for writes.
|
||||
async fn put_bucket_object(
|
||||
State(reg): State<Arc<BucketRegistry>>,
|
||||
Path((bucket, key)): Path<(String, String)>,
|
||||
body: bytes::Bytes,
|
||||
) -> impl IntoResponse {
|
||||
match reg.write_smart(&bucket, &key, body).await {
|
||||
Ok(()) => (StatusCode::CREATED, format!("stored: {bucket}/{key}")).into_response(),
|
||||
Err(e) => (StatusCode::SERVICE_UNAVAILABLE, e).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bucket-aware read. Falls through to the rescue bucket if the target
|
||||
/// is unreachable or the key is missing; emits observability headers so
|
||||
/// callers can detect the fallback.
|
||||
async fn flush_errors(State(reg): State<Arc<BucketRegistry>>) -> impl IntoResponse {
|
||||
match reg.journal().flush().await {
|
||||
Ok(()) => (StatusCode::OK, "flushed").into_response(),
|
||||
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn compact_errors(State(reg): State<Arc<BucketRegistry>>) -> impl IntoResponse {
|
||||
match reg.journal().compact().await {
|
||||
Ok(stats) => Json(stats).into_response(),
|
||||
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_bucket_object(
|
||||
State(reg): State<Arc<BucketRegistry>>,
|
||||
Path((bucket, key)): Path<(String, String)>,
|
||||
) -> impl IntoResponse {
|
||||
match reg.read_smart(&bucket, &key).await {
|
||||
Ok(outcome) => {
|
||||
let mut resp = outcome.data.into_response();
|
||||
if outcome.rescued {
|
||||
let headers = resp.headers_mut();
|
||||
headers.insert("x-lakehouse-rescue-used", "true".parse().unwrap());
|
||||
headers.insert(
|
||||
"x-lakehouse-original-bucket",
|
||||
outcome.original_bucket.parse().unwrap(),
|
||||
);
|
||||
headers.insert(
|
||||
"x-lakehouse-served-by",
|
||||
outcome.served_by.parse().unwrap(),
|
||||
);
|
||||
}
|
||||
resp
|
||||
}
|
||||
Err(e) => (StatusCode::NOT_FOUND, e).into_response(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
pub mod append_log;
|
||||
pub mod backend;
|
||||
pub mod error_journal;
|
||||
pub mod federation_service;
|
||||
pub mod ops;
|
||||
pub mod registry;
|
||||
pub mod service;
|
||||
@@ -0,0 +1,30 @@
|
||||
use bytes::Bytes;
|
||||
use futures::TryStreamExt;
|
||||
use object_store::{ObjectStore, path::Path};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub async fn put(store: &Arc<dyn ObjectStore>, key: &str, data: Bytes) -> Result<(), String> {
|
||||
let path = Path::from(key);
|
||||
store.put(&path, data.into()).await.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get(store: &Arc<dyn ObjectStore>, key: &str) -> Result<Bytes, String> {
|
||||
let path = Path::from(key);
|
||||
let result = store.get(&path).await.map_err(|e| e.to_string())?;
|
||||
let bytes = result.bytes().await.map_err(|e| e.to_string())?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub async fn delete(store: &Arc<dyn ObjectStore>, key: &str) -> Result<(), String> {
|
||||
let path = Path::from(key);
|
||||
store.delete(&path).await.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list(store: &Arc<dyn ObjectStore>, prefix: Option<&str>) -> Result<Vec<String>, String> {
|
||||
let prefix_path = prefix.map(Path::from);
|
||||
let stream = store.list(prefix_path.as_ref());
|
||||
let items: Vec<_> = stream.try_collect().await.map_err(|e| e.to_string())?;
|
||||
Ok(items.into_iter().map(|m| m.location.to_string()).collect())
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
/// Multi-backend bucket registry — the federation foundation.
|
||||
///
|
||||
/// Federation rule: every `ObjectRef` belongs to exactly one named bucket.
|
||||
/// The registry resolves bucket names to `object_store` backends, handles
|
||||
/// rescue-bucket fallback on read failure, writes every failure to the
|
||||
/// error journal, and exposes a health summary for operators.
|
||||
///
|
||||
/// Existing call sites can keep using `ops::*` with `registry.get(name)`.
|
||||
/// New bucket-aware call sites use `registry.read_smart` / `write_smart`
|
||||
/// which handle fallback + journaling automatically.
|
||||
|
||||
use object_store::ObjectStore;
|
||||
use object_store::local::LocalFileSystem;
|
||||
use serde::Serialize;
|
||||
use shared::config::{BucketConfig, StorageConfig};
|
||||
use shared::secrets::{BucketCredentials, SecretsProvider};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use crate::error_journal::{BucketErrorEvent, ErrorJournal};
|
||||
|
||||
/// A registered bucket — the store handle + its configuration.
|
||||
pub struct BucketEntry {
|
||||
pub name: String,
|
||||
pub backend: String,
|
||||
pub store: Arc<dyn ObjectStore>,
|
||||
pub config: BucketConfig,
|
||||
}
|
||||
|
||||
/// Read outcome — may have been rescued.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReadOutcome {
|
||||
pub data: bytes::Bytes,
|
||||
pub rescued: bool,
|
||||
pub original_bucket: String,
|
||||
pub served_by: String,
|
||||
}
|
||||
|
||||
/// Summary entry for GET /storage/buckets.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct BucketInfo {
|
||||
pub name: String,
|
||||
pub backend: String,
|
||||
pub reachable: bool,
|
||||
pub role: BucketRole,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum BucketRole {
|
||||
Primary,
|
||||
Rescue,
|
||||
Profile,
|
||||
Tenant,
|
||||
}
|
||||
|
||||
pub struct BucketRegistry {
|
||||
/// RwLock because federation layer 2 adds runtime bucket lifecycle
|
||||
/// (`POST /storage/buckets` / `DELETE /storage/buckets/{name}`).
|
||||
/// Almost all accesses are reads — writes happen at provision /
|
||||
/// deactivate time only.
|
||||
buckets: RwLock<HashMap<String, Arc<BucketEntry>>>,
|
||||
default: String,
|
||||
rescue: Option<String>,
|
||||
profile_root: String,
|
||||
/// Held so runtime `add_bucket` can resolve secret_ref handles.
|
||||
secrets: Arc<dyn SecretsProvider>,
|
||||
journal: ErrorJournal,
|
||||
}
|
||||
|
||||
impl BucketRegistry {
|
||||
/// Build the registry from storage config + secrets provider.
|
||||
/// Back-compat: if `buckets` is empty, synthesize a `primary` bucket from
|
||||
/// the legacy `root` field so pre-federation configs keep working.
|
||||
pub async fn from_config(
|
||||
cfg: &StorageConfig,
|
||||
secrets: Arc<dyn SecretsProvider>,
|
||||
) -> Result<Self, String> {
|
||||
let mut buckets: HashMap<String, Arc<BucketEntry>> = HashMap::new();
|
||||
|
||||
let bucket_configs: Vec<BucketConfig> = if cfg.buckets.is_empty() {
|
||||
vec![BucketConfig {
|
||||
name: "primary".to_string(),
|
||||
backend: "local".to_string(),
|
||||
root: Some(cfg.root.clone()),
|
||||
bucket: None,
|
||||
region: None,
|
||||
endpoint: None,
|
||||
secret_ref: None,
|
||||
}]
|
||||
} else {
|
||||
cfg.buckets.clone()
|
||||
};
|
||||
|
||||
for bc in bucket_configs {
|
||||
let store = build_store(&bc, secrets.as_ref()).await?;
|
||||
let entry = Arc::new(BucketEntry {
|
||||
name: bc.name.clone(),
|
||||
backend: bc.backend.clone(),
|
||||
store,
|
||||
config: bc.clone(),
|
||||
});
|
||||
buckets.insert(bc.name.clone(), entry);
|
||||
}
|
||||
|
||||
// Ensure `primary` always exists — it's where error journals live.
|
||||
if !buckets.contains_key("primary") {
|
||||
return Err("no bucket named 'primary' configured — required as error-journal home".into());
|
||||
}
|
||||
|
||||
// Rescue bucket is optional but, if named, must exist.
|
||||
if let Some(r) = &cfg.rescue_bucket {
|
||||
if !buckets.contains_key(r) {
|
||||
return Err(format!("rescue_bucket '{r}' not found among configured buckets"));
|
||||
}
|
||||
}
|
||||
|
||||
let journal = ErrorJournal::new(buckets.get("primary").unwrap().store.clone());
|
||||
let _ = journal.load_recent().await;
|
||||
|
||||
Ok(Self {
|
||||
buckets: RwLock::new(buckets),
|
||||
default: "primary".to_string(),
|
||||
rescue: cfg.rescue_bucket.clone(),
|
||||
profile_root: cfg.profile_root.clone(),
|
||||
secrets,
|
||||
journal,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn default_name(&self) -> &str { &self.default }
|
||||
|
||||
pub fn rescue_name(&self) -> Option<&str> { self.rescue.as_deref() }
|
||||
|
||||
pub fn profile_root(&self) -> &str { &self.profile_root }
|
||||
|
||||
pub fn journal(&self) -> &ErrorJournal { &self.journal }
|
||||
|
||||
/// Resolve a bucket name to its object store. Existing call sites use
|
||||
/// this as a drop-in replacement for the old single-store pattern.
|
||||
///
|
||||
/// Uses `std::sync::RwLock` — every caller holds the guard for
|
||||
/// microseconds (clones an Arc and drops), so there's no async-
|
||||
/// blocking concern. Never hold across an await.
|
||||
pub fn get(&self, bucket: &str) -> Result<Arc<dyn ObjectStore>, String> {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard
|
||||
.get(bucket)
|
||||
.map(|e| e.store.clone())
|
||||
.ok_or_else(|| format!("unknown bucket: {bucket}"))
|
||||
}
|
||||
|
||||
/// The default bucket's store — use for code paths that don't yet know
|
||||
/// about buckets.
|
||||
pub fn default_store(&self) -> Arc<dyn ObjectStore> {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard.get(&self.default).unwrap().store.clone()
|
||||
}
|
||||
|
||||
/// True if this bucket name is registered.
|
||||
pub fn contains(&self, bucket: &str) -> bool {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard.contains_key(bucket)
|
||||
}
|
||||
|
||||
/// List all registered buckets. Checks reachability by doing a trivial
|
||||
/// `list` with limit 1 on each.
|
||||
pub async fn list(&self) -> Vec<BucketInfo> {
|
||||
// Snapshot (name, backend, store_clone) under the lock, then probe
|
||||
// outside — probing can be slow (network), don't hold the lock.
|
||||
let snapshot: Vec<(String, String, Arc<dyn ObjectStore>)> = {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard
|
||||
.iter()
|
||||
.map(|(n, e)| (n.clone(), e.backend.clone(), e.store.clone()))
|
||||
.collect()
|
||||
};
|
||||
|
||||
let mut out = Vec::with_capacity(snapshot.len());
|
||||
for (name, backend, store) in snapshot {
|
||||
let reachable = probe(&store).await;
|
||||
let role = self.classify(&name);
|
||||
out.push(BucketInfo { name, backend, reachable, role });
|
||||
}
|
||||
out.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
out
|
||||
}
|
||||
|
||||
fn classify(&self, name: &str) -> BucketRole {
|
||||
if name == self.default { BucketRole::Primary }
|
||||
else if Some(name) == self.rescue.as_deref() { BucketRole::Rescue }
|
||||
else if name.starts_with("profile:") { BucketRole::Profile }
|
||||
else { BucketRole::Tenant }
|
||||
}
|
||||
|
||||
/// Provision + register a bucket at runtime (federation layer 2).
|
||||
/// Returns Err if the name is already registered. Local backends get
|
||||
/// their root directory created automatically.
|
||||
pub async fn add_bucket(&self, bc: BucketConfig) -> Result<BucketInfo, String> {
|
||||
if self.contains(&bc.name) {
|
||||
return Err(format!("bucket '{}' already registered", bc.name));
|
||||
}
|
||||
let store = build_store(&bc, self.secrets.as_ref()).await?;
|
||||
let reachable = probe(&store).await;
|
||||
let entry = Arc::new(BucketEntry {
|
||||
name: bc.name.clone(),
|
||||
backend: bc.backend.clone(),
|
||||
store,
|
||||
config: bc.clone(),
|
||||
});
|
||||
let role = self.classify(&bc.name);
|
||||
let info = BucketInfo {
|
||||
name: bc.name.clone(),
|
||||
backend: bc.backend.clone(),
|
||||
reachable,
|
||||
role,
|
||||
};
|
||||
{
|
||||
let mut guard = self.buckets.write().expect("bucket registry lock poisoned");
|
||||
guard.insert(bc.name.clone(), entry);
|
||||
}
|
||||
tracing::info!("registered bucket '{}' backend={} reachable={}",
|
||||
bc.name, bc.backend, reachable);
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
/// Unregister a bucket. Refuses to remove `primary` or the configured
|
||||
/// rescue bucket — those are load-bearing. Caller is responsible for
|
||||
/// any final flush of bucket-local state before calling this.
|
||||
pub fn remove_bucket(&self, name: &str) -> Result<(), String> {
|
||||
if name == self.default {
|
||||
return Err("cannot remove primary bucket".into());
|
||||
}
|
||||
if Some(name) == self.rescue.as_deref() {
|
||||
return Err(format!("cannot remove rescue bucket '{name}'"));
|
||||
}
|
||||
let removed = {
|
||||
let mut guard = self.buckets.write().expect("bucket registry lock poisoned");
|
||||
guard.remove(name)
|
||||
};
|
||||
if removed.is_none() {
|
||||
return Err(format!("bucket '{name}' not registered"));
|
||||
}
|
||||
tracing::info!("unregistered bucket '{}'", name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read with rescue-bucket fallback. If the target bucket fails and a
|
||||
/// rescue is configured, retries against rescue. Records every failure
|
||||
/// in the error journal.
|
||||
pub async fn read_smart(&self, bucket: &str, key: &str) -> Result<ReadOutcome, String> {
|
||||
let target_store = {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard.get(bucket).map(|e| e.store.clone())
|
||||
.ok_or_else(|| format!("unknown bucket: {bucket}"))?
|
||||
};
|
||||
|
||||
match crate::ops::get(&target_store, key).await {
|
||||
Ok(data) => Ok(ReadOutcome {
|
||||
data, rescued: false,
|
||||
original_bucket: bucket.to_string(),
|
||||
served_by: bucket.to_string(),
|
||||
}),
|
||||
Err(err) => {
|
||||
// Record failure regardless of what happens next.
|
||||
self.journal.append(BucketErrorEvent::new_read(bucket, key, &err)).await;
|
||||
|
||||
// Try rescue, if any.
|
||||
if let Some(rescue_name) = &self.rescue {
|
||||
if rescue_name != bucket {
|
||||
let rescue_store = {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard.get(rescue_name).map(|e| e.store.clone())
|
||||
};
|
||||
if let Some(rescue_store) = rescue_store {
|
||||
match crate::ops::get(&rescue_store, key).await {
|
||||
Ok(data) => {
|
||||
self.journal.mark_rescued_last(bucket, key).await;
|
||||
return Ok(ReadOutcome {
|
||||
data, rescued: true,
|
||||
original_bucket: bucket.to_string(),
|
||||
served_by: rescue_name.clone(),
|
||||
});
|
||||
}
|
||||
Err(rescue_err) => {
|
||||
return Err(format!(
|
||||
"read '{key}' failed in '{bucket}' ({err}); rescue '{rescue_name}' also failed ({rescue_err})"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!("read '{key}' failed in '{bucket}': {err}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write always goes to target. No rescue fallback for writes — writes
|
||||
/// that silently vanish are the worst possible failure.
|
||||
pub async fn write_smart(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
data: bytes::Bytes,
|
||||
) -> Result<(), String> {
|
||||
let target_store = {
|
||||
let guard = self.buckets.read().expect("bucket registry lock poisoned");
|
||||
guard.get(bucket).map(|e| e.store.clone())
|
||||
.ok_or_else(|| format!("unknown bucket: {bucket}"))?
|
||||
};
|
||||
match crate::ops::put(&target_store, key, data).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) => {
|
||||
self.journal.append(BucketErrorEvent::new_write(bucket, key, &err)).await;
|
||||
Err(format!("write '{key}' failed in '{bucket}': {err}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trivial reachability check — try to list with limit 0.
|
||||
async fn probe(store: &Arc<dyn ObjectStore>) -> bool {
|
||||
use futures::StreamExt;
|
||||
let mut stream = store.list(None);
|
||||
// Pulling the first item confirms the store responds. Empty bucket = ok.
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => true,
|
||||
None => true, // empty but reachable
|
||||
Some(Err(_)) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a concrete ObjectStore from a BucketConfig.
|
||||
async fn build_store(
|
||||
bc: &BucketConfig,
|
||||
secrets: &dyn SecretsProvider,
|
||||
) -> Result<Arc<dyn ObjectStore>, String> {
|
||||
match bc.backend.as_str() {
|
||||
"local" => {
|
||||
let root = bc.root.as_deref()
|
||||
.ok_or_else(|| format!("bucket '{}' is backend=local but has no root", bc.name))?;
|
||||
std::fs::create_dir_all(root)
|
||||
.map_err(|e| format!("create bucket dir '{root}': {e}"))?;
|
||||
let fs = LocalFileSystem::new_with_prefix(root)
|
||||
.map_err(|e| format!("init local bucket '{}': {e}", bc.name))?;
|
||||
Ok(Arc::new(fs))
|
||||
}
|
||||
"s3" => {
|
||||
let handle = bc.secret_ref.as_deref()
|
||||
.ok_or_else(|| format!("s3 bucket '{}' has no secret_ref", bc.name))?;
|
||||
let creds: BucketCredentials = secrets.resolve(handle).await?;
|
||||
let s3_bucket = bc.bucket.as_deref()
|
||||
.ok_or_else(|| format!("s3 bucket '{}' has no `bucket` name", bc.name))?;
|
||||
let region = bc.region.as_deref().unwrap_or("us-east-1");
|
||||
|
||||
let mut builder = object_store::aws::AmazonS3Builder::new()
|
||||
.with_bucket_name(s3_bucket)
|
||||
.with_region(region)
|
||||
.with_access_key_id(&creds.access_key)
|
||||
.with_secret_access_key(&creds.secret_key);
|
||||
if let Some(endpoint) = &bc.endpoint {
|
||||
builder = builder.with_endpoint(endpoint);
|
||||
// MinIO and other S3-compatible services often run on plain
|
||||
// HTTP. object_store refuses HTTP by default — opt in when
|
||||
// a custom endpoint is configured (TLS endpoints work either way).
|
||||
if endpoint.starts_with("http://") {
|
||||
builder = builder.with_allow_http(true);
|
||||
}
|
||||
}
|
||||
let s3 = builder.build()
|
||||
.map_err(|e| format!("init s3 bucket '{}': {e}", bc.name))?;
|
||||
Ok(Arc::new(s3))
|
||||
}
|
||||
other => Err(format!("unknown backend '{other}' for bucket '{}'", bc.name)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{delete, get, put},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use object_store::ObjectStore;
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::ops;
|
||||
|
||||
pub type StorageState = Arc<dyn ObjectStore>;
|
||||
|
||||
pub fn router(store: Arc<dyn ObjectStore>) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/objects/{*key}", put(put_object))
|
||||
.route("/objects/{*key}", get(get_object))
|
||||
.route("/objects/{*key}", delete(delete_object))
|
||||
.route("/objects", get(list_objects))
|
||||
.with_state(store)
|
||||
}
|
||||
|
||||
async fn health() -> &'static str {
|
||||
"storaged ok"
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
State(store): State<StorageState>,
|
||||
Path(key): Path<String>,
|
||||
body: Bytes,
|
||||
) -> impl IntoResponse {
|
||||
match ops::put(&store, &key, body).await {
|
||||
Ok(()) => (StatusCode::CREATED, format!("stored: {key}")),
|
||||
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object(
|
||||
State(store): State<StorageState>,
|
||||
Path(key): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match ops::get(&store, &key).await {
|
||||
Ok(data) => Ok(data),
|
||||
Err(e) => Err((StatusCode::NOT_FOUND, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(
|
||||
State(store): State<StorageState>,
|
||||
Path(key): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match ops::delete(&store, &key).await {
|
||||
Ok(()) => (StatusCode::OK, format!("deleted: {key}")),
|
||||
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ListQuery {
|
||||
prefix: Option<String>,
|
||||
}
|
||||
|
||||
async fn list_objects(
|
||||
State(store): State<StorageState>,
|
||||
Query(q): Query<ListQuery>,
|
||||
) -> impl IntoResponse {
|
||||
match ops::list(&store, q.prefix.as_deref()).await {
|
||||
Ok(keys) => Ok(axum::Json(keys)),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user