Phase 11: Embedding versioning — model-proof vector layer

- IndexRegistry: tracks all vector indexes with model metadata
  (model_name, model_version, dimensions, build stats)
- Index metadata persisted as JSON in vectors/meta/
- Rebuilt on startup for crash recovery
- GET /vectors/indexes — list all indexes (filter by source/model)
- GET /vectors/indexes/{name} — get index metadata
- Background jobs auto-register metadata on completion
- Multi-version support: same data, different models, coexist
- Per ADR-014: enables incremental re-embed on model upgrade

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 09:27:10 -05:00
co-authored by Claude Opus 4.6
parent 6d49f81ebf
commit 6cd1daeb51
4 changed files with 185 additions and 10 deletions
+112
View File
@@ -0,0 +1,112 @@
/// Vector index registry — tracks all indexes with model versioning.
/// Each index knows which model created it, enabling:
/// - Multi-version indexes (same data, different models, coexist)
/// - Incremental re-embed (only new/changed docs on model upgrade)
/// - A/B search comparison between model versions
use chrono::{DateTime, Utc};
use object_store::ObjectStore;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use storaged::ops;
/// Metadata for a vector index.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexMeta {
pub index_name: String,
pub source: String, // dataset this was built from
pub model_name: String, // "nomic-embed-text"
pub model_version: String, // "latest" or specific version
pub dimensions: u32, // 768
pub chunk_count: usize,
pub doc_count: usize,
pub chunk_size: usize,
pub overlap: usize,
pub storage_key: String, // "vectors/resumes_v1_nomic.parquet"
pub created_at: DateTime<Utc>,
pub build_time_secs: f32,
pub chunks_per_sec: f32,
}
/// Registry of all vector indexes.
#[derive(Clone)]
pub struct IndexRegistry {
indexes: Arc<RwLock<HashMap<String, IndexMeta>>>,
store: Arc<dyn ObjectStore>,
}
impl IndexRegistry {
pub fn new(store: Arc<dyn ObjectStore>) -> Self {
Self {
indexes: Arc::new(RwLock::new(HashMap::new())),
store,
}
}
/// Rebuild from persisted index metadata on startup.
pub async fn rebuild(&self) -> Result<usize, String> {
let keys = ops::list(&self.store, Some("vectors/meta/")).await?;
let mut reg = self.indexes.write().await;
reg.clear();
for key in &keys {
if !key.ends_with(".json") { continue; }
let data = ops::get(&self.store, key).await?;
match serde_json::from_slice::<IndexMeta>(&data) {
Ok(meta) => { reg.insert(meta.index_name.clone(), meta); }
Err(e) => tracing::warn!("failed to load index meta {key}: {e}"),
}
}
let count = reg.len();
if count > 0 {
tracing::info!("loaded {count} vector index metadata entries");
}
Ok(count)
}
/// Register a new index.
pub async fn register(&self, meta: IndexMeta) -> Result<(), String> {
let key = format!("vectors/meta/{}.json", meta.index_name);
let json = serde_json::to_vec_pretty(&meta).map_err(|e| e.to_string())?;
ops::put(&self.store, &key, json.into()).await?;
self.indexes.write().await.insert(meta.index_name.clone(), meta);
Ok(())
}
/// Get metadata for an index.
pub async fn get(&self, index_name: &str) -> Option<IndexMeta> {
self.indexes.read().await.get(index_name).cloned()
}
/// List all indexes, optionally filtered by source or model.
pub async fn list(&self, source: Option<&str>, model: Option<&str>) -> Vec<IndexMeta> {
self.indexes.read().await.values()
.filter(|m| source.map_or(true, |s| m.source == s))
.filter(|m| model.map_or(true, |mo| m.model_name == mo))
.cloned()
.collect()
}
/// Find all versions of an index for a given source dataset.
/// Returns indexes sorted by creation time (newest first).
pub async fn versions_for_source(&self, source: &str) -> Vec<IndexMeta> {
let mut versions: Vec<IndexMeta> = self.indexes.read().await.values()
.filter(|m| m.source == source)
.cloned()
.collect();
versions.sort_by(|a, b| b.created_at.cmp(&a.created_at));
versions
}
/// Delete an index (metadata only — vector Parquet stays for safety).
pub async fn delete(&self, index_name: &str) -> Result<(), String> {
let key = format!("vectors/meta/{index_name}.json");
ops::delete(&self.store, &key).await?;
self.indexes.write().await.remove(index_name);
Ok(())
}
}