profit/lakehouse
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Phase 28-36 body of work

Browse Source 
Accumulated since a6f12e2 (Phase 21 Rust port + Phase 27 versioning):

- Phase 36: embed_semaphore on VectorState (permits=1) serializes
  seed embed calls — prevents sidecar socket collisions under
  concurrent /seed stress load
- Phase 31+: run_stress.ts 6-task diverse stress scaffolding;
  run_e2e_rated.ts + orchestrator.ts tightening
- Catalog dedupe cleanup: 16 duplicate manifests removed; canonical
  candidates.parquet (10.5MB -> 76KB) + placements.parquet (1.2MB ->
  11KB) regenerated post-dedupe; fresh manifests for active datasets
- vectord: harness EvalSet refinements (+181), agent portfolio
  rotation + ingest triggers (+158), autotune + rag adjustments
- catalogd/storaged/ingestd/mcp-server: misc tightening
- docs: Phase 28-36 PRD entries + DECISIONS ADR additions;
  control-plane pivot banner added to top of docs/PRD.md (pointing
  at docs/CONTROL_PLANE_PRD.md which lands in next commit)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
profit
2026-04-22 02:41:15 -05:00
co-authored by Claude Opus 4.7
parent a6f12e2609
commit 5b1fcf6d27
52 changed files with 2575 additions and 1012 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/vectord/src/agent.rs
+118 -40
View File
@@ -204,6 +204,7 @@ pub struct AgentDeps {
pub embedding_cache: EmbeddingCache,
pub trial_journal: TrialJournal,
pub promotion_registry: PromotionRegistry,
pub harness_store: crate::harness::HarnessStore,
}
// -------- Spawn --------
@@ -354,12 +355,11 @@ async fn run_one_cycle(
// Read history.
let history = deps.trial_journal.list(&event.index_name).await
.map_err(|e| format!("read journal: {e}"))?;
if history.is_empty() {
return Err(format!(
"no trials yet for '{}' — seed with at least one POST /hnsw/trial first",
event.index_name,
));
}
// Bootstrap mode: first ever visit to this index. Auto-generate a
// harness (if missing) and seed with the default config — the Phase 15
// known-good ec=80/es=30. Subsequent visits go through the proposer.
let is_bootstrap = history.is_empty();
// Current champion (if any) is the promoted config.
let champion = deps.promotion_registry.get_current(&event.index_name).await;
@@ -367,24 +367,26 @@ async fn run_one_cycle(
history.iter().find(|t| t.id == p.trial_id).cloned()
});
// Propose the next config.
let Some(next_config) = propose_next_config(&history, champion_trial.as_ref()) else {
return Err("proposer returned None — search space exhausted".into());
let (next_config, harness_name) = if is_bootstrap {
let name = ensure_auto_harness(&event.index_name, deps).await?;
(HnswConfig::default(), name)
} else {
let Some(cfg) = propose_next_config(&history, champion_trial.as_ref()) else {
return Err("proposer returned None — search space exhausted".into());
};
// Validate bounds defensively.
if !(10..=400).contains(&cfg.ef_construction) {
return Err(format!("proposed ef_construction={} out of bounds", cfg.ef_construction));
}
if !(10..=200).contains(&cfg.ef_search) {
return Err(format!("proposed ef_search={} out of bounds", cfg.ef_search));
}
// A future refinement: remember per-index "canonical harness" on
// the index metadata. For now: latest wins.
let hname = history.last().unwrap().eval_set.clone();
(cfg, hname)
};
// Validate bounds defensively.
if !(10..=400).contains(&next_config.ef_construction) {
return Err(format!("proposed ef_construction={} out of bounds", next_config.ef_construction));
}
if !(10..=200).contains(&next_config.ef_search) {
return Err(format!("proposed ef_search={} out of bounds", next_config.ef_search));
}
// Need a harness to measure. Use the most recent one in history.
// (A future refinement: remember per-index "canonical harness" on
// the index metadata. For now: latest wins.)
let harness_name = history.last().unwrap().eval_set.clone();
let mut harness_set = harness::EvalSet::load(&deps.store, &harness_name).await
let mut harness_set = deps.harness_store.load_for_index(&event.index_name, &harness_name).await
.map_err(|e| format!("load harness '{harness_name}': {e}"))?;
let embeddings = deps.embedding_cache.get_or_load(&event.index_name).await
@@ -393,7 +395,7 @@ async fn run_one_cycle(
if !harness_set.ground_truth_built {
harness::compute_ground_truth(&mut harness_set, &embeddings, &deps.ai_client).await
.map_err(|e| format!("ground truth: {e}"))?;
harness_set.save(&deps.store).await.ok();
deps.harness_store.save(&harness_set).await.ok();
}
// Build + bench.
@@ -428,10 +430,13 @@ async fn run_one_cycle(
let dims = embeddings.first().map(|e| e.vector.len()).unwrap_or(0);
let memory_bytes = (embeddings.len() * dims * std::mem::size_of::<f32>() + embeddings.len() * 128) as u64;
let note = match &event.reason {
TriggerReason::DatasetAppended { dataset } => format!("agent: dataset_appended({dataset})"),
TriggerReason::Manual => "agent: manual".to_string(),
TriggerReason::Periodic => "agent: periodic".to_string(),
let note = {
let base = match &event.reason {
TriggerReason::DatasetAppended { dataset } => format!("agent: dataset_appended({dataset})"),
TriggerReason::Manual => "agent: manual".to_string(),
TriggerReason::Periodic => "agent: periodic".to_string(),
};
if is_bootstrap { format!("{base} bootstrap") } else { base }
};
let trial = Trial {
@@ -597,19 +602,92 @@ pub fn propose_next_config(history: &[Trial], champion: Option<&Trial>) -> Optio
// -------- Helpers --------
/// Find an index to poke on a periodic wake. Strategy: the one with the
/// most recent trial. If nothing's been trialed yet, return None.
/// Minimum vectors for an index to be worth auto-tuning. Below this,
/// HNSW's win over brute-force is too small to matter and trial budget
/// is better spent on bigger indexes.
const AUTOTUNE_MIN_VECTORS: usize = 1_000;
/// Find an index to poke on a periodic wake. Strategy: least-recently-tuned
/// wins — pick the index (from `IndexRegistry`, not just promoted ones)
/// whose most-recent trial is oldest, so trial budget spreads across the
/// whole portfolio. Never-trialed indexes sort first (None < Some), which
/// is exactly what we want: bootstrap them on their first visit.
///
/// Why not "most recently promoted" (the original strategy): a converged
/// index like `threat_intel_v1` sits at recall=1.0 on a minimal graph and
/// can't be improved — but it was always the freshest promotion, so the
/// agent burned every trial on it while 40+ other indexes got zero
/// attention. Rotating by last-trial-time fixes that without any explicit
/// convergence detection.
///
/// Why IndexRegistry instead of PromotionRegistry: on a fresh system only
/// a handful of indexes have ever been promoted, so promotion-based
/// picking starves new ones. IndexRegistry lists every live index the
/// platform knows about; filtering by `AUTOTUNE_MIN_VECTORS` keeps the
/// proposer off indexes where the graph config doesn't matter.
///
/// Cost: N extra journal reads per periodic tick, where N = eligible
/// indexes. Fine at dozens; if the portfolio grows into thousands, cache
/// `last_trial_at` on the IndexMeta.
async fn pick_periodic_target(deps: &AgentDeps) -> Option<String> {
// `/agent` runs against any index that has a trial journal. We don't
// have a "list all journals" helper, so we derive candidates from the
// promotion registry (indexes with a human ever promoting are live).
let promos = deps.promotion_registry.list_all().await.ok()?;
// Prefer the one most recently promoted — it's the one a human cares
// about right now.
promos.into_iter()
.filter_map(|f| f.current.map(|c| (f.index_name, c.promoted_at)))
.max_by_key(|(_, at)| *at)
.map(|(name, _)| name)
let candidates: Vec<String> = deps
.index_registry
.list(None, None)
.await
.into_iter()
.filter(|m| m.chunk_count >= AUTOTUNE_MIN_VECTORS)
.map(|m| m.index_name)
.collect();
if candidates.is_empty() { return None; }
let mut scored: Vec<(String, Option<DateTime<Utc>>)> = Vec::with_capacity(candidates.len());
for name in candidates {
let last_trial_at = deps
.trial_journal
.list(&name)
.await
.ok()
.and_then(|trials| trials.into_iter().map(|t| t.created_at).max());
scored.push((name, last_trial_at));
}
scored.into_iter().min_by_key(|(_, last)| *last).map(|(n, _)| n)
}
/// On the first visit to an index, load or synthesize an eval harness so
/// bootstrap trials have something to measure recall against. Returns the
/// harness name. Synthetic harnesses sample 20 chunks and use their first
/// 200 chars as self-queries — the ground truth is computed by brute-force
/// cosine, so recall numbers are real (a config that finds the chunk in
/// top-k counts as a hit).
async fn ensure_auto_harness(index_name: &str, deps: &AgentDeps) -> Result<String, String> {
let name = format!("{index_name}_auto");
if deps.harness_store.load_for_index(index_name, &name).await.is_ok() {
return Ok(name);
}
let embeddings = deps
.embedding_cache
.get_or_load(index_name)
.await
.map_err(|e| format!("load embeddings for auto-harness: {e}"))?;
if embeddings.is_empty() {
return Err(format!("index '{index_name}' has no embeddings — cannot bootstrap"));
}
let mut eval = harness::synthetic_from_chunks(&name, index_name, &embeddings, 20, 10);
harness::compute_ground_truth(&mut eval, &embeddings, &deps.ai_client)
.await
.map_err(|e| format!("auto-harness ground truth: {e}"))?;
deps.harness_store.save(&eval).await
.map_err(|e| format!("save auto-harness: {e}"))?;
tracing::info!(
"agent: bootstrapped harness '{}' for index '{}' (20 self-queries, k=10)",
name, index_name
);
Ok(name)
}
async fn over_rate_limit(inner: &Arc<AgentInner>, cap: u32) -> bool {