Federation foundation + HNSW trial system + Postgres streaming + PRD reframe

Four shipped features and a PRD realignment, all measured end-to-end: HNSW trial system (Phase 15 horizon item → complete) - vectord: EmbeddingCache, harness (eval sets + brute-force ground truth), TrialJournal, parameterized HnswConfig on build_index_with_config - /vectors/hnsw/trial, /hnsw/trials/{idx}, /hnsw/trials/{idx}/best, /hnsw/evals/{name}/autogen, /hnsw/cache/stats - Measured on resumes_100k_v2 (100K × 768d): brute-force 44ms -> HNSW 873us at 100% recall@10. ec=80 es=30 locked as HnswConfig::default() - Lower ec values trade recall for build time: 20/30 = 0.96 recall in 8s, 80/30 = 1.00 recall in 230s Catalog manifest repair - catalogd: resync_from_parquet reads parquet footers to restore row_count and columns on drifted manifests - POST /catalog/datasets/{name}/resync + POST /catalog/resync-missing - All 7 staffing tables recovered to PRD-matching 2,469,278 rows Federation foundation (ADR-017) - shared::secrets: SecretsProvider trait + FileSecretsProvider (reads /etc/lakehouse/secrets.toml, enforces 0600 perms) - storaged::registry::BucketRegistry — multi-bucket resolution with rescue_bucket read fallback and reachability probing - storaged::error_journal — bucket op failures visible in one HTTP call - storaged::append_log — write-once batched append pattern (fixes the RMW anti-pattern llms3.com calls out; errors and trial journals both use it) - /storage/buckets, /storage/errors, /storage/bucket-health, /storage/errors/{flush,compact} - Bucket-aware I/O at /storage/buckets/{bucket}/objects/{*key} with X-Lakehouse-Rescue-Used observability headers on fallback Postgres streaming ingest - ingestd::pg_stream: DSN parser, batched ORDER BY + LIMIT/OFFSET pagination into ArrowWriter, lineage redacts password - POST /ingest/db — verified against live knowledge_base.team_runs (586 rows × 13 cols, 6 batches, 196ms end-to-end) PRD realignment (2026-04-16) - Dual use case: staffing analytics + local LLM knowledge substrate - Removed "multi-tenancy (single-owner system)" from non-goals - Added invariants 8-11: indexes hot-swappable, per-reader profiles, trials-as-data, operational failures findable in one HTTP call - New phases 16 (hot-swap generations), 17 (model profiles + dataset bindings), 18 (Lance vs Parquet+sidecar evaluation) - Known ceilings table documents the 5M vector wall and escape hatches - ADR-017 (federation), ADR-018 (append-log pattern) added - EXECUTION_PLAN.md sequences phases B-E with success gates and decision rules Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-16 01:50:05 -05:00
parent 84407eeb51
commit dbe00d018f
33 changed files with 3305 additions and 52 deletions
@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use std::sync::Arc;

 use aibridge::client::{AiClient, EmbedRequest};
-use crate::{chunker, hnsw, index_registry, jobs, rag, search, store, supervisor};
+use crate::{chunker, embedding_cache, harness, hnsw, index_registry, jobs, rag, search, store, supervisor, trial};

 #[derive(Clone)]
 pub struct VectorState {
@@ -19,6 +19,8 @@ pub struct VectorState {
    pub job_tracker: jobs::JobTracker,
    pub index_registry: index_registry::IndexRegistry,
    pub hnsw_store: hnsw::HnswStore,
+    pub embedding_cache: embedding_cache::EmbeddingCache,
+    pub trial_journal: trial::TrialJournal,
 }

 pub fn router(state: VectorState) -> Router {
@@ -34,6 +36,17 @@ pub fn router(state: VectorState) -> Router {
        .route("/hnsw/build", post(build_hnsw))
        .route("/hnsw/search", post(search_hnsw))
        .route("/hnsw/list", get(list_hnsw))
+        // Trial system — parameterized tuning loop
+        .route("/hnsw/trial", post(run_trial))
+        .route("/hnsw/trials/{index_name}", get(list_trials))
+        .route("/hnsw/trials/{index_name}/best", get(best_trial))
+        // Eval sets
+        .route("/hnsw/evals", get(list_evals))
+        .route("/hnsw/evals/{name}", get(get_eval).put(put_eval))
+        .route("/hnsw/evals/{name}/autogen", post(autogen_eval))
+        // Cache management
+        .route("/hnsw/cache/stats", get(cache_stats))
+        .route("/hnsw/cache/{index_name}", axum::routing::delete(cache_evict))
        .with_state(state)
 }

@@ -307,26 +320,35 @@ async fn rag_query(
 struct BuildHnswRequest {
    /// Name of the stored vector index to build HNSW from
    index_name: String,
+    /// Optional config override. Omit to use the production default
+    /// (ec=80 es=30 — see HnswConfig::default docs for rationale).
+    #[serde(default)]
+    config: Option<trial::HnswConfig>,
 }

 /// Build an HNSW index from an existing stored vector index.
-/// Loads embeddings from Parquet, builds HNSW in memory.
+/// Uses the embedding cache so repeated builds don't reload from Parquet.
 async fn build_hnsw(
    State(state): State<VectorState>,
    Json(req): Json<BuildHnswRequest>,
 ) -> impl IntoResponse {
-    tracing::info!("building HNSW for '{}'", req.index_name);
+    let config = req.config.unwrap_or_default();
+    tracing::info!(
+        "building HNSW for '{}' ef_construction={} ef_search={}",
+        req.index_name, config.ef_construction, config.ef_search,
+    );

-    // Load embeddings from Parquet
-    let embeddings = store::load_embeddings(&state.store, &req.index_name)
+    let embeddings = state
+        .embedding_cache
+        .get_or_load(&req.index_name)
        .await
        .map_err(|e| (StatusCode::NOT_FOUND, format!("index not found: {e}")))?;

-    let n = embeddings.len();
-    tracing::info!("loaded {} embeddings, building HNSW...", n);
-
-    // Build HNSW
-    match state.hnsw_store.build_index(&req.index_name, embeddings).await {
+    match state
+        .hnsw_store
+        .build_index_with_config(&req.index_name, (*embeddings).clone(), &config)
+        .await
+    {
        Ok(stats) => Ok(Json(stats)),
        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
    }
@@ -372,3 +394,275 @@ async fn search_hnsw(
 async fn list_hnsw(State(state): State<VectorState>) -> impl IntoResponse {
    Json(state.hnsw_store.list().await)
 }
+
+// --- Trial System: parameterized HNSW tuning loop ---
+//
+// Flow:
+//   1. Agent picks an HnswConfig
+//   2. POST /hnsw/trial builds HNSW with that config against cached embeddings,
+//      runs every query in the harness, measures latency + recall vs the
+//      harness's ground truth, appends a Trial record to _hnsw_trials/{idx}.jsonl
+//   3. Agent reads GET /hnsw/trials/{index}, sees history, decides next config
+//   4. Repeat until converged.
+//
+// The first trial triggers embedding load (slow). Every subsequent trial reuses
+// the cache — so the agent iterates in seconds, not minutes.
+
+#[derive(Deserialize)]
+struct TrialRequest {
+    index_name: String,
+    harness: String,
+    #[serde(default)]
+    config: trial::HnswConfig,
+    #[serde(default)]
+    note: Option<String>,
+}
+
+async fn run_trial(
+    State(state): State<VectorState>,
+    Json(req): Json<TrialRequest>,
+) -> Result<Json<trial::Trial>, (StatusCode, String)> {
+    let mut harness_set = harness::EvalSet::load(&state.store, &req.harness)
+        .await
+        .map_err(|e| (StatusCode::NOT_FOUND, format!("harness not found: {e}")))?;
+
+    if harness_set.index_name != req.index_name {
+        return Err((
+            StatusCode::BAD_REQUEST,
+            format!(
+                "harness '{}' is for index '{}', not '{}'",
+                req.harness, harness_set.index_name, req.index_name
+            ),
+        ));
+    }
+    if harness_set.queries.is_empty() {
+        return Err((StatusCode::BAD_REQUEST, "harness has no queries".into()));
+    }
+
+    let embeddings = state
+        .embedding_cache
+        .get_or_load(&req.index_name)
+        .await
+        .map_err(|e| (StatusCode::NOT_FOUND, format!("load embeddings: {e}")))?;
+
+    if !harness_set.ground_truth_built {
+        tracing::info!("trial: computing ground truth for harness '{}'", harness_set.name);
+        let t0 = std::time::Instant::now();
+        harness::compute_ground_truth(&mut harness_set, &embeddings, &state.ai_client)
+            .await
+            .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("ground truth: {e}")))?;
+        tracing::info!("trial: ground truth built in {:.1}s", t0.elapsed().as_secs_f32());
+        harness_set
+            .save(&state.store)
+            .await
+            .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("save harness: {e}")))?;
+    }
+
+    let trial_id = trial::Trial::new_id();
+    let hnsw_slot = format!("{}__{}", req.index_name, trial_id);
+
+    let build_stats = state
+        .hnsw_store
+        .build_index_with_config(&hnsw_slot, (*embeddings).clone(), &req.config)
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("build: {e}")))?;
+
+    let query_vectors: Vec<Vec<f32>> = harness_set
+        .queries
+        .iter()
+        .filter_map(|q| q.query_embedding.clone())
+        .collect();
+    let bench = state
+        .hnsw_store
+        .bench_search(&hnsw_slot, &query_vectors, harness_set.k)
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("search: {e}")))?;
+
+    let mut recalls = Vec::with_capacity(harness_set.queries.len());
+    for (q, hits) in harness_set.queries.iter().zip(bench.retrieved.iter()) {
+        if let Some(gt) = &q.ground_truth {
+            recalls.push(harness::recall_at_k(hits, gt, harness_set.k));
+        }
+    }
+    let mean_recall = if recalls.is_empty() {
+        0.0
+    } else {
+        recalls.iter().sum::<f32>() / recalls.len() as f32
+    };
+
+    let mut lats = bench.latencies_us.clone();
+    lats.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+    let p = |pct: f32| -> f32 {
+        if lats.is_empty() { return 0.0; }
+        let idx = ((lats.len() as f32 - 1.0) * pct).round() as usize;
+        lats[idx.min(lats.len() - 1)]
+    };
+
+    // One brute-force reference latency — keeps the cost proportional to
+    // whatever the agent is willing to pay per trial.
+    let brute_latency_us = if let Some(qv) = query_vectors.first() {
+        let t0 = std::time::Instant::now();
+        let _ = harness::brute_force_top_k(qv, &embeddings, harness_set.k);
+        t0.elapsed().as_micros() as f32
+    } else {
+        0.0
+    };
+
+    let dims = embeddings.first().map(|e| e.vector.len()).unwrap_or(0);
+    let memory_bytes =
+        (embeddings.len() * dims * std::mem::size_of::<f32>() + embeddings.len() * 128) as u64;
+
+    let trial_record = trial::Trial {
+        id: trial_id.clone(),
+        index_name: req.index_name.clone(),
+        eval_set: req.harness.clone(),
+        config: req.config.clone(),
+        metrics: trial::TrialMetrics {
+            build_time_secs: build_stats.build_time_secs,
+            search_latency_p50_us: p(0.50),
+            search_latency_p95_us: p(0.95),
+            search_latency_p99_us: p(0.99),
+            recall_at_k: mean_recall,
+            memory_bytes,
+            vectors: build_stats.vectors,
+            eval_queries: harness_set.queries.len(),
+            brute_force_latency_us: brute_latency_us,
+        },
+        created_at: chrono::Utc::now(),
+        note: req.note,
+    };
+
+    state
+        .trial_journal
+        .append(&trial_record)
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("journal: {e}")))?;
+
+    state.hnsw_store.drop(&hnsw_slot).await;
+
+    Ok(Json(trial_record))
+}
+
+async fn list_trials(
+    State(state): State<VectorState>,
+    Path(index_name): Path<String>,
+) -> impl IntoResponse {
+    match state.trial_journal.list(&index_name).await {
+        Ok(trials) => Ok(Json(trials)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct BestTrialQuery {
+    #[serde(default = "default_metric")]
+    metric: String,
+}
+
+fn default_metric() -> String {
+    "pareto".to_string()
+}
+
+async fn best_trial(
+    State(state): State<VectorState>,
+    Path(index_name): Path<String>,
+    Query(q): Query<BestTrialQuery>,
+) -> impl IntoResponse {
+    match state.trial_journal.best(&index_name, &q.metric).await {
+        Ok(Some(t)) => Ok(Json(t)),
+        Ok(None) => Err((StatusCode::NOT_FOUND, "no trials yet".to_string())),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+// --- Harness management ---
+
+async fn list_evals(State(state): State<VectorState>) -> impl IntoResponse {
+    match harness::EvalSet::list(&state.store).await {
+        Ok(names) => Ok(Json(names)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn get_eval(
+    State(state): State<VectorState>,
+    Path(name): Path<String>,
+) -> impl IntoResponse {
+    match harness::EvalSet::load(&state.store, &name).await {
+        Ok(e) => Ok(Json(e)),
+        Err(err) => Err((StatusCode::NOT_FOUND, err)),
+    }
+}
+
+async fn put_eval(
+    State(state): State<VectorState>,
+    Path(name): Path<String>,
+    Json(mut harness_set): Json<harness::EvalSet>,
+) -> impl IntoResponse {
+    harness_set.name = name;
+    harness_set.ground_truth_built = harness_set
+        .queries
+        .iter()
+        .all(|q| q.ground_truth.is_some());
+    match harness_set.save(&state.store).await {
+        Ok(()) => Ok(Json(harness_set)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+#[derive(Deserialize)]
+struct AutogenRequest {
+    index_name: String,
+    #[serde(default = "default_sample_count")]
+    sample_count: usize,
+    #[serde(default = "default_k")]
+    k: usize,
+}
+
+fn default_sample_count() -> usize { 100 }
+fn default_k() -> usize { 10 }
+
+async fn autogen_eval(
+    State(state): State<VectorState>,
+    Path(name): Path<String>,
+    Json(req): Json<AutogenRequest>,
+) -> Result<Json<harness::EvalSet>, (StatusCode, String)> {
+    let embeddings = state
+        .embedding_cache
+        .get_or_load(&req.index_name)
+        .await
+        .map_err(|e| (StatusCode::NOT_FOUND, format!("load embeddings: {e}")))?;
+
+    let mut harness_set = harness::synthetic_from_chunks(
+        &name,
+        &req.index_name,
+        &embeddings,
+        req.sample_count,
+        req.k,
+    );
+
+    harness::compute_ground_truth(&mut harness_set, &embeddings, &state.ai_client)
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("ground truth: {e}")))?;
+
+    harness_set
+        .save(&state.store)
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("save: {e}")))?;
+
+    Ok(Json(harness_set))
+}
+
+// --- Embedding cache management ---
+
+async fn cache_stats(State(state): State<VectorState>) -> impl IntoResponse {
+    Json(state.embedding_cache.stats().await)
+}
+
+async fn cache_evict(
+    State(state): State<VectorState>,
+    Path(index_name): Path<String>,
+) -> impl IntoResponse {
+    let ok = state.embedding_cache.evict(&index_name).await;
+    Json(serde_json::json!({ "evicted": ok, "index_name": index_name }))
+}