Phase 28-36 body of work
Accumulated since a6f12e2 (Phase 21 Rust port + Phase 27 versioning):
- Phase 36: embed_semaphore on VectorState (permits=1) serializes
seed embed calls — prevents sidecar socket collisions under
concurrent /seed stress load
- Phase 31+: run_stress.ts 6-task diverse stress scaffolding;
run_e2e_rated.ts + orchestrator.ts tightening
- Catalog dedupe cleanup: 16 duplicate manifests removed; canonical
candidates.parquet (10.5MB -> 76KB) + placements.parquet (1.2MB ->
11KB) regenerated post-dedupe; fresh manifests for active datasets
- vectord: harness EvalSet refinements (+181), agent portfolio
rotation + ingest triggers (+158), autotune + rag adjustments
- catalogd/storaged/ingestd/mcp-server: misc tightening
- docs: Phase 28-36 PRD entries + DECISIONS ADR additions;
control-plane pivot banner added to top of docs/PRD.md (pointing
at docs/CONTROL_PLANE_PRD.md which lands in next commit)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
a6f12e2609
commit
5b1fcf6d27
+118
-40
@@ -204,6 +204,7 @@ pub struct AgentDeps {
|
||||
pub embedding_cache: EmbeddingCache,
|
||||
pub trial_journal: TrialJournal,
|
||||
pub promotion_registry: PromotionRegistry,
|
||||
pub harness_store: crate::harness::HarnessStore,
|
||||
}
|
||||
|
||||
// -------- Spawn --------
|
||||
@@ -354,12 +355,11 @@ async fn run_one_cycle(
|
||||
// Read history.
|
||||
let history = deps.trial_journal.list(&event.index_name).await
|
||||
.map_err(|e| format!("read journal: {e}"))?;
|
||||
if history.is_empty() {
|
||||
return Err(format!(
|
||||
"no trials yet for '{}' — seed with at least one POST /hnsw/trial first",
|
||||
event.index_name,
|
||||
));
|
||||
}
|
||||
|
||||
// Bootstrap mode: first ever visit to this index. Auto-generate a
|
||||
// harness (if missing) and seed with the default config — the Phase 15
|
||||
// known-good ec=80/es=30. Subsequent visits go through the proposer.
|
||||
let is_bootstrap = history.is_empty();
|
||||
|
||||
// Current champion (if any) is the promoted config.
|
||||
let champion = deps.promotion_registry.get_current(&event.index_name).await;
|
||||
@@ -367,24 +367,26 @@ async fn run_one_cycle(
|
||||
history.iter().find(|t| t.id == p.trial_id).cloned()
|
||||
});
|
||||
|
||||
// Propose the next config.
|
||||
let Some(next_config) = propose_next_config(&history, champion_trial.as_ref()) else {
|
||||
return Err("proposer returned None — search space exhausted".into());
|
||||
let (next_config, harness_name) = if is_bootstrap {
|
||||
let name = ensure_auto_harness(&event.index_name, deps).await?;
|
||||
(HnswConfig::default(), name)
|
||||
} else {
|
||||
let Some(cfg) = propose_next_config(&history, champion_trial.as_ref()) else {
|
||||
return Err("proposer returned None — search space exhausted".into());
|
||||
};
|
||||
// Validate bounds defensively.
|
||||
if !(10..=400).contains(&cfg.ef_construction) {
|
||||
return Err(format!("proposed ef_construction={} out of bounds", cfg.ef_construction));
|
||||
}
|
||||
if !(10..=200).contains(&cfg.ef_search) {
|
||||
return Err(format!("proposed ef_search={} out of bounds", cfg.ef_search));
|
||||
}
|
||||
// A future refinement: remember per-index "canonical harness" on
|
||||
// the index metadata. For now: latest wins.
|
||||
let hname = history.last().unwrap().eval_set.clone();
|
||||
(cfg, hname)
|
||||
};
|
||||
|
||||
// Validate bounds defensively.
|
||||
if !(10..=400).contains(&next_config.ef_construction) {
|
||||
return Err(format!("proposed ef_construction={} out of bounds", next_config.ef_construction));
|
||||
}
|
||||
if !(10..=200).contains(&next_config.ef_search) {
|
||||
return Err(format!("proposed ef_search={} out of bounds", next_config.ef_search));
|
||||
}
|
||||
|
||||
// Need a harness to measure. Use the most recent one in history.
|
||||
// (A future refinement: remember per-index "canonical harness" on
|
||||
// the index metadata. For now: latest wins.)
|
||||
let harness_name = history.last().unwrap().eval_set.clone();
|
||||
let mut harness_set = harness::EvalSet::load(&deps.store, &harness_name).await
|
||||
let mut harness_set = deps.harness_store.load_for_index(&event.index_name, &harness_name).await
|
||||
.map_err(|e| format!("load harness '{harness_name}': {e}"))?;
|
||||
|
||||
let embeddings = deps.embedding_cache.get_or_load(&event.index_name).await
|
||||
@@ -393,7 +395,7 @@ async fn run_one_cycle(
|
||||
if !harness_set.ground_truth_built {
|
||||
harness::compute_ground_truth(&mut harness_set, &embeddings, &deps.ai_client).await
|
||||
.map_err(|e| format!("ground truth: {e}"))?;
|
||||
harness_set.save(&deps.store).await.ok();
|
||||
deps.harness_store.save(&harness_set).await.ok();
|
||||
}
|
||||
|
||||
// Build + bench.
|
||||
@@ -428,10 +430,13 @@ async fn run_one_cycle(
|
||||
let dims = embeddings.first().map(|e| e.vector.len()).unwrap_or(0);
|
||||
let memory_bytes = (embeddings.len() * dims * std::mem::size_of::<f32>() + embeddings.len() * 128) as u64;
|
||||
|
||||
let note = match &event.reason {
|
||||
TriggerReason::DatasetAppended { dataset } => format!("agent: dataset_appended({dataset})"),
|
||||
TriggerReason::Manual => "agent: manual".to_string(),
|
||||
TriggerReason::Periodic => "agent: periodic".to_string(),
|
||||
let note = {
|
||||
let base = match &event.reason {
|
||||
TriggerReason::DatasetAppended { dataset } => format!("agent: dataset_appended({dataset})"),
|
||||
TriggerReason::Manual => "agent: manual".to_string(),
|
||||
TriggerReason::Periodic => "agent: periodic".to_string(),
|
||||
};
|
||||
if is_bootstrap { format!("{base} bootstrap") } else { base }
|
||||
};
|
||||
|
||||
let trial = Trial {
|
||||
@@ -597,19 +602,92 @@ pub fn propose_next_config(history: &[Trial], champion: Option<&Trial>) -> Optio
|
||||
|
||||
// -------- Helpers --------
|
||||
|
||||
/// Find an index to poke on a periodic wake. Strategy: the one with the
|
||||
/// most recent trial. If nothing's been trialed yet, return None.
|
||||
/// Minimum vectors for an index to be worth auto-tuning. Below this,
|
||||
/// HNSW's win over brute-force is too small to matter and trial budget
|
||||
/// is better spent on bigger indexes.
|
||||
const AUTOTUNE_MIN_VECTORS: usize = 1_000;
|
||||
|
||||
/// Find an index to poke on a periodic wake. Strategy: least-recently-tuned
|
||||
/// wins — pick the index (from `IndexRegistry`, not just promoted ones)
|
||||
/// whose most-recent trial is oldest, so trial budget spreads across the
|
||||
/// whole portfolio. Never-trialed indexes sort first (None < Some), which
|
||||
/// is exactly what we want: bootstrap them on their first visit.
|
||||
///
|
||||
/// Why not "most recently promoted" (the original strategy): a converged
|
||||
/// index like `threat_intel_v1` sits at recall=1.0 on a minimal graph and
|
||||
/// can't be improved — but it was always the freshest promotion, so the
|
||||
/// agent burned every trial on it while 40+ other indexes got zero
|
||||
/// attention. Rotating by last-trial-time fixes that without any explicit
|
||||
/// convergence detection.
|
||||
///
|
||||
/// Why IndexRegistry instead of PromotionRegistry: on a fresh system only
|
||||
/// a handful of indexes have ever been promoted, so promotion-based
|
||||
/// picking starves new ones. IndexRegistry lists every live index the
|
||||
/// platform knows about; filtering by `AUTOTUNE_MIN_VECTORS` keeps the
|
||||
/// proposer off indexes where the graph config doesn't matter.
|
||||
///
|
||||
/// Cost: N extra journal reads per periodic tick, where N = eligible
|
||||
/// indexes. Fine at dozens; if the portfolio grows into thousands, cache
|
||||
/// `last_trial_at` on the IndexMeta.
|
||||
async fn pick_periodic_target(deps: &AgentDeps) -> Option<String> {
|
||||
// `/agent` runs against any index that has a trial journal. We don't
|
||||
// have a "list all journals" helper, so we derive candidates from the
|
||||
// promotion registry (indexes with a human ever promoting are live).
|
||||
let promos = deps.promotion_registry.list_all().await.ok()?;
|
||||
// Prefer the one most recently promoted — it's the one a human cares
|
||||
// about right now.
|
||||
promos.into_iter()
|
||||
.filter_map(|f| f.current.map(|c| (f.index_name, c.promoted_at)))
|
||||
.max_by_key(|(_, at)| *at)
|
||||
.map(|(name, _)| name)
|
||||
let candidates: Vec<String> = deps
|
||||
.index_registry
|
||||
.list(None, None)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|m| m.chunk_count >= AUTOTUNE_MIN_VECTORS)
|
||||
.map(|m| m.index_name)
|
||||
.collect();
|
||||
if candidates.is_empty() { return None; }
|
||||
|
||||
let mut scored: Vec<(String, Option<DateTime<Utc>>)> = Vec::with_capacity(candidates.len());
|
||||
for name in candidates {
|
||||
let last_trial_at = deps
|
||||
.trial_journal
|
||||
.list(&name)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|trials| trials.into_iter().map(|t| t.created_at).max());
|
||||
scored.push((name, last_trial_at));
|
||||
}
|
||||
|
||||
scored.into_iter().min_by_key(|(_, last)| *last).map(|(n, _)| n)
|
||||
}
|
||||
|
||||
/// On the first visit to an index, load or synthesize an eval harness so
|
||||
/// bootstrap trials have something to measure recall against. Returns the
|
||||
/// harness name. Synthetic harnesses sample 20 chunks and use their first
|
||||
/// 200 chars as self-queries — the ground truth is computed by brute-force
|
||||
/// cosine, so recall numbers are real (a config that finds the chunk in
|
||||
/// top-k counts as a hit).
|
||||
async fn ensure_auto_harness(index_name: &str, deps: &AgentDeps) -> Result<String, String> {
|
||||
let name = format!("{index_name}_auto");
|
||||
|
||||
if deps.harness_store.load_for_index(index_name, &name).await.is_ok() {
|
||||
return Ok(name);
|
||||
}
|
||||
|
||||
let embeddings = deps
|
||||
.embedding_cache
|
||||
.get_or_load(index_name)
|
||||
.await
|
||||
.map_err(|e| format!("load embeddings for auto-harness: {e}"))?;
|
||||
if embeddings.is_empty() {
|
||||
return Err(format!("index '{index_name}' has no embeddings — cannot bootstrap"));
|
||||
}
|
||||
|
||||
let mut eval = harness::synthetic_from_chunks(&name, index_name, &embeddings, 20, 10);
|
||||
harness::compute_ground_truth(&mut eval, &embeddings, &deps.ai_client)
|
||||
.await
|
||||
.map_err(|e| format!("auto-harness ground truth: {e}"))?;
|
||||
deps.harness_store.save(&eval).await
|
||||
.map_err(|e| format!("save auto-harness: {e}"))?;
|
||||
|
||||
tracing::info!(
|
||||
"agent: bootstrapped harness '{}' for index '{}' (20 self-queries, k=10)",
|
||||
name, index_name
|
||||
);
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
async fn over_rate_limit(inner: &Arc<AgentInner>, cap: u32) -> bool {
|
||||
|
||||
@@ -28,7 +28,7 @@ use aibridge::client::AiClient;
|
||||
use catalogd::registry::Registry as CatalogRegistry;
|
||||
|
||||
use crate::embedding_cache::EmbeddingCache;
|
||||
use crate::harness;
|
||||
use crate::harness::{self, HarnessStore};
|
||||
use crate::hnsw::HnswStore;
|
||||
use crate::index_registry::IndexRegistry;
|
||||
use crate::jobs::JobTracker;
|
||||
@@ -128,7 +128,7 @@ fn pick_winner(trials: &[Trial], min_recall: f32) -> Option<&Trial> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_autotune(
|
||||
req: AutotuneRequest,
|
||||
store: &Arc<dyn ObjectStore>,
|
||||
_store: &Arc<dyn ObjectStore>,
|
||||
catalog: &CatalogRegistry,
|
||||
ai_client: &AiClient,
|
||||
embedding_cache: &EmbeddingCache,
|
||||
@@ -136,6 +136,7 @@ pub async fn run_autotune(
|
||||
index_registry: &IndexRegistry,
|
||||
trial_journal: &TrialJournal,
|
||||
promotion_registry: &PromotionRegistry,
|
||||
harness_store: &HarnessStore,
|
||||
_job_tracker: &JobTracker,
|
||||
) -> Result<AutotuneResult, String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
@@ -146,8 +147,11 @@ pub async fn run_autotune(
|
||||
}
|
||||
let _ = catalog; // reserved for future audit emission
|
||||
|
||||
// Load the harness once, compute ground truth once.
|
||||
let mut harness_set = harness::EvalSet::load(store, &req.harness)
|
||||
// Load the harness once, compute ground truth once. Harness resolves
|
||||
// to the index's bucket via HarnessStore, with primary as fallback for
|
||||
// pre-federation evals.
|
||||
let mut harness_set = harness_store
|
||||
.load_for_index(&req.index_name, &req.harness)
|
||||
.await
|
||||
.map_err(|e| format!("load harness: {e}"))?;
|
||||
|
||||
@@ -160,7 +164,7 @@ pub async fn run_autotune(
|
||||
harness::compute_ground_truth(&mut harness_set, &embeddings, ai_client)
|
||||
.await
|
||||
.map_err(|e| format!("ground truth: {e}"))?;
|
||||
harness_set.save(store).await.ok();
|
||||
harness_store.save(&harness_set).await.ok();
|
||||
}
|
||||
|
||||
let (grid, rejected) = sanitize_grid(req.grid.clone().unwrap_or_else(default_grid));
|
||||
|
||||
@@ -10,11 +10,14 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use object_store::ObjectStore;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use aibridge::client::{AiClient, EmbedRequest};
|
||||
use storaged::ops;
|
||||
use storaged::registry::BucketRegistry;
|
||||
|
||||
use crate::index_registry::IndexRegistry;
|
||||
use crate::store::StoredEmbedding;
|
||||
|
||||
/// A single eval query with optional pre-computed ground truth.
|
||||
@@ -85,6 +88,108 @@ impl EvalSet {
|
||||
}
|
||||
}
|
||||
|
||||
/// Federation-aware wrapper around EvalSet persistence. Mirrors the
|
||||
/// `TrialJournal` / `PromotionRegistry` pattern: harness files colocate
|
||||
/// with their index's bucket (looked up via `IndexMeta.bucket`), falling
|
||||
/// back to `primary` for indexes the registry has never seen. Legacy
|
||||
/// harnesses predating federation remain discoverable — lookups transparently
|
||||
/// try the resolved bucket first, then `primary` as a fallback. Cross-bucket
|
||||
/// listing dedupes across every registered bucket so `GET /hnsw/evals`
|
||||
/// returns a complete picture.
|
||||
#[derive(Clone)]
|
||||
pub struct HarnessStore {
|
||||
buckets: Arc<BucketRegistry>,
|
||||
index_registry: IndexRegistry,
|
||||
}
|
||||
|
||||
impl HarnessStore {
|
||||
pub fn new(buckets: Arc<BucketRegistry>, index_registry: IndexRegistry) -> Self {
|
||||
Self { buckets, index_registry }
|
||||
}
|
||||
|
||||
/// Resolve which bucket holds this index's eval artifacts. Indexes the
|
||||
/// registry has never heard of fall through to `primary`.
|
||||
async fn bucket_for_index(&self, index_name: &str) -> String {
|
||||
self.index_registry
|
||||
.get(index_name)
|
||||
.await
|
||||
.map(|m| m.bucket)
|
||||
.unwrap_or_else(|| "primary".to_string())
|
||||
}
|
||||
|
||||
/// Save to the bucket that owns `eval.index_name`. Writes under the
|
||||
/// standard `_hnsw_evals/{name}.json` prefix of the resolved bucket.
|
||||
pub async fn save(&self, eval: &EvalSet) -> Result<(), String> {
|
||||
let bucket = self.bucket_for_index(&eval.index_name).await;
|
||||
let store = self.buckets.get(&bucket)?;
|
||||
eval.save(&store).await
|
||||
}
|
||||
|
||||
/// Load a harness by name, given the index it belongs to. Tries the
|
||||
/// index's bucket first; if the file is absent AND the resolved bucket
|
||||
/// isn't `primary`, falls through to `primary` so pre-federation evals
|
||||
/// remain reachable without migration.
|
||||
pub async fn load_for_index(
|
||||
&self,
|
||||
index_name: &str,
|
||||
harness_name: &str,
|
||||
) -> Result<EvalSet, String> {
|
||||
let bucket = self.bucket_for_index(index_name).await;
|
||||
let primary_store = self.buckets.get("primary")?;
|
||||
let store = self.buckets.get(&bucket)?;
|
||||
|
||||
match EvalSet::load(&store, harness_name).await {
|
||||
Ok(e) => Ok(e),
|
||||
Err(e) if bucket != "primary" => EvalSet::load(&primary_store, harness_name)
|
||||
.await
|
||||
.map_err(|primary_err| format!("{bucket}: {e}; primary fallback: {primary_err}")),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a harness by name without knowing which index it belongs to —
|
||||
/// used by `GET /hnsw/evals/{name}`. Scans every registered bucket;
|
||||
/// first hit wins. Primary is searched first so pre-federation evals
|
||||
/// with the same name as a federated one resolve deterministically.
|
||||
pub async fn get_any(&self, harness_name: &str) -> Result<EvalSet, String> {
|
||||
let bucket_infos = self.buckets.list().await;
|
||||
let mut ordered: Vec<String> = bucket_infos.iter().map(|b| b.name.clone()).collect();
|
||||
ordered.sort_by_key(|n| if n == "primary" { 0 } else { 1 });
|
||||
|
||||
let mut last_err = None;
|
||||
for b in ordered {
|
||||
let store = match self.buckets.get(&b) {
|
||||
Ok(s) => s,
|
||||
Err(e) => { last_err = Some(e); continue; }
|
||||
};
|
||||
match EvalSet::load(&store, harness_name).await {
|
||||
Ok(e) => return Ok(e),
|
||||
Err(e) => { last_err = Some(e); }
|
||||
}
|
||||
}
|
||||
Err(last_err.unwrap_or_else(|| format!("no buckets registered for eval '{harness_name}'")))
|
||||
}
|
||||
|
||||
/// Union of every harness name across every registered bucket.
|
||||
/// Duplicates (same name in multiple buckets — pathological but
|
||||
/// possible after a manual migration) are collapsed.
|
||||
pub async fn list_all(&self) -> Vec<String> {
|
||||
let mut all: HashSet<String> = HashSet::new();
|
||||
for b in self.buckets.list().await {
|
||||
let store = match self.buckets.get(&b.name) {
|
||||
Ok(s) => s,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if let Ok(names) = EvalSet::list(&store).await {
|
||||
all.extend(names);
|
||||
}
|
||||
}
|
||||
let mut out: Vec<String> = all.into_iter().collect();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Cosine similarity for two same-length f32 slices.
|
||||
fn cosine(a: &[f32], b: &[f32]) -> f32 {
|
||||
let mut dot = 0.0f32;
|
||||
@@ -127,13 +232,15 @@ pub fn recall_at_k(predicted: &[String], ground_truth: &[String], k: usize) -> f
|
||||
if ground_truth.is_empty() || k == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
// Set-intersection recall@k. Previous implementation counted duplicates
|
||||
// in `predicted` (a corpus with repeated chunks — e.g. cached LLM
|
||||
// responses — returns the same doc_id multiple times via HNSW), which
|
||||
// inflated recall above 1.0 and poisoned promotion decisions.
|
||||
let gt_set: std::collections::HashSet<&String> =
|
||||
ground_truth.iter().take(k).collect();
|
||||
let hits = predicted
|
||||
.iter()
|
||||
.take(k)
|
||||
.filter(|d| gt_set.contains(d))
|
||||
.count();
|
||||
let pred_set: std::collections::HashSet<&String> =
|
||||
predicted.iter().take(k).collect();
|
||||
let hits = pred_set.intersection(>_set).count();
|
||||
hits as f32 / gt_set.len() as f32
|
||||
}
|
||||
|
||||
@@ -214,3 +321,67 @@ pub fn synthetic_from_chunks(
|
||||
ground_truth_built: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn s(v: &[&str]) -> Vec<String> { v.iter().map(|x| x.to_string()).collect() }
|
||||
|
||||
#[test]
|
||||
fn recall_empty_ground_truth_is_zero() {
|
||||
assert_eq!(recall_at_k(&s(&["a", "b"]), &[], 10), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_k_zero_is_zero() {
|
||||
assert_eq!(recall_at_k(&s(&["a"]), &s(&["a"]), 0), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_perfect_match_equals_one() {
|
||||
let pred = s(&["a", "b", "c"]);
|
||||
let gt = s(&["a", "b", "c"]);
|
||||
assert!((recall_at_k(&pred, >, 3) - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_half_match() {
|
||||
let pred = s(&["a", "b", "x", "y"]);
|
||||
let gt = s(&["a", "b", "c", "d"]);
|
||||
assert!((recall_at_k(&pred, >, 4) - 0.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_duplicates_in_predicted_do_not_inflate() {
|
||||
// Regression guard: the previous implementation counted each
|
||||
// duplicate in `predicted` separately, which could push recall
|
||||
// above 1.0 on corpora with repeated chunks (cached responses etc).
|
||||
// Set-intersection semantics keep it bounded in [0, 1].
|
||||
let pred = s(&["a", "a", "a", "a"]);
|
||||
let gt = s(&["a", "b", "c", "d"]);
|
||||
let r = recall_at_k(&pred, >, 4);
|
||||
assert!(r <= 1.0, "recall {r} must not exceed 1.0");
|
||||
// One unique match out of four in gt = 0.25.
|
||||
assert!((r - 0.25).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_duplicates_in_ground_truth_handled() {
|
||||
// gt with dupes reduces effective |gt|; matching one hits all.
|
||||
let pred = s(&["x"]);
|
||||
let gt = s(&["x", "x", "x"]);
|
||||
let r = recall_at_k(&pred, >, 3);
|
||||
assert!(r <= 1.0);
|
||||
assert!((r - 1.0).abs() < 1e-6); // |pred ∩ gt| / |gt_set| = 1/1
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_respects_k_bound() {
|
||||
// k=2 means only the first 2 of pred and gt count toward the set.
|
||||
let pred = s(&["a", "b", "c", "d"]);
|
||||
let gt = s(&["a", "b", "c", "d"]);
|
||||
let r = recall_at_k(&pred, >, 2);
|
||||
assert!((r - 1.0).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
|
||||
+26
-12
@@ -10,6 +10,7 @@ use object_store::ObjectStore;
|
||||
use std::sync::Arc;
|
||||
|
||||
use aibridge::client::{AiClient, EmbedRequest, GenerateRequest};
|
||||
use aibridge::continuation::{generate_continuable, ContinuableOpts, ResponseShape};
|
||||
use crate::search::{self, SearchResult};
|
||||
use crate::store;
|
||||
|
||||
@@ -42,7 +43,10 @@ async fn rerank(
|
||||
system: None,
|
||||
temperature: Some(0.0),
|
||||
max_tokens: Some(50),
|
||||
think: None,
|
||||
// Reranker returns a comma-separated int list — pure structured
|
||||
// output, zero benefit from hidden reasoning. Opt out to avoid
|
||||
// the empty-response failure mode Phase 21 catalogued.
|
||||
think: Some(false),
|
||||
}).await;
|
||||
|
||||
match resp {
|
||||
@@ -151,19 +155,29 @@ pub async fn query(
|
||||
Answer:"
|
||||
);
|
||||
|
||||
let gen_resp = ai_client.generate(GenerateRequest {
|
||||
prompt,
|
||||
model: None,
|
||||
system: None,
|
||||
temperature: Some(0.2),
|
||||
max_tokens: Some(512),
|
||||
think: None,
|
||||
}).await?;
|
||||
// Route the answer call through Phase 21's generate_continuable so
|
||||
// a thinking-model empty-response or a mid-JSON truncation self-
|
||||
// repairs instead of silently returning half an answer. Shape is
|
||||
// Text (the answer is prose, not JSON), think is Some(false) to
|
||||
// opt out of hidden reasoning on the hot path. This is the first
|
||||
// production caller of the Phase 21 primitives — see audit finding
|
||||
// "Phase 21 Rust primitives are wired but not CALLED by any
|
||||
// production surface" from 2026-04-21.
|
||||
let mut cont_opts = ContinuableOpts::new("qwen2.5:latest");
|
||||
cont_opts.max_tokens = Some(512);
|
||||
cont_opts.temperature = Some(0.2);
|
||||
cont_opts.shape = ResponseShape::Text;
|
||||
cont_opts.think = Some(false);
|
||||
let outcome = generate_continuable(ai_client, &prompt, &cont_opts).await?;
|
||||
|
||||
Ok(RagResponse {
|
||||
answer: gen_resp.text.trim().to_string(),
|
||||
model: gen_resp.model,
|
||||
answer: outcome.text.trim().to_string(),
|
||||
// generate_continuable doesn't surface the model name (sidecar
|
||||
// echoes whatever Ollama loaded). Use the configured tier model
|
||||
// for now; if RAG needs to report the actual resolved model,
|
||||
// the runner can add a post-call ps probe later.
|
||||
model: "qwen2.5:latest".to_string(),
|
||||
sources: results,
|
||||
tokens_generated: gen_resp.tokens_generated,
|
||||
tokens_generated: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ use aibridge::client::{AiClient, EmbedRequest, GenerateRequest};
|
||||
use catalogd::registry::Registry as CatalogRegistry;
|
||||
use storaged::registry::BucketRegistry;
|
||||
use crate::{agent, autotune, chunker, embedding_cache, harness, hnsw, index_registry, jobs, lance_backend, playbook_memory, promotion, rag, refresh, search, store, supervisor, trial};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct VectorState {
|
||||
@@ -53,6 +54,9 @@ pub struct VectorState {
|
||||
/// and, when `use_playbook_memory` is set on /vectors/hybrid, boosts
|
||||
/// workers that were actually filled in semantically-similar past ops.
|
||||
pub playbook_memory: playbook_memory::PlaybookMemory,
|
||||
/// Serializes embed calls from seed_playbook_memory to avoid
|
||||
/// concurrent socket collisions with the Python sidecar.
|
||||
pub embed_semaphore: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
/// What the active-profile singleton records. Narrow — we don't need the
|
||||
@@ -893,7 +897,9 @@ async fn hybrid_search(
|
||||
system: None,
|
||||
temperature: Some(0.2),
|
||||
max_tokens: Some(512),
|
||||
think: None,
|
||||
// Hybrid's answer step — prose output over retrieved records,
|
||||
// no reasoning needed on the hot path.
|
||||
think: Some(false),
|
||||
}).await;
|
||||
|
||||
gen_resp.ok().map(|r| r.text.trim().to_string())
|
||||
@@ -1583,7 +1589,7 @@ async fn activate_profile(
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Json(ActivateReport {
|
||||
let report = ActivateReport {
|
||||
profile_id: profile.id,
|
||||
ollama_name: profile.ollama_name,
|
||||
indexes_warmed: warmed,
|
||||
@@ -1592,7 +1598,9 @@ async fn activate_profile(
|
||||
duration_secs: t0.elapsed().as_secs_f32(),
|
||||
model_preloaded,
|
||||
previous_profile: previous_slot.map(|s| s.profile_id),
|
||||
}))
|
||||
};
|
||||
|
||||
Ok(Json(report))
|
||||
}
|
||||
|
||||
/// Unload this profile's model and clear the active slot. No-op if the
|
||||
@@ -2217,6 +2225,10 @@ async fn seed_playbook_memory(
|
||||
State(state): State<VectorState>,
|
||||
Json(req): Json<SeedPlaybookRequest>,
|
||||
) -> impl IntoResponse {
|
||||
// Serialize embed calls to avoid concurrent socket collisions.
|
||||
let _permit = state.embed_semaphore.acquire().await.map_err(|e|
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, format!("semaphore error: {e}")))?;
|
||||
|
||||
// Embed the entry through the same text shape `rebuild` uses so
|
||||
// similarity math is comparable across seed + real entries.
|
||||
let tmp_entry = playbook_memory::PlaybookEntry {
|
||||
@@ -2549,17 +2561,47 @@ async fn revise_playbook_memory(
|
||||
"operation must match 'fill: Role xN in City, ST' shape".into()));
|
||||
}
|
||||
|
||||
let ts = chrono::Utc::now().to_rfc3339();
|
||||
// Phase 27 — deterministic pid derived ONLY from content-shaped
|
||||
// inputs (parent_id + operation + approach + context + sorted
|
||||
// endorsed_names). Excluding wall-clock ts means two revise calls
|
||||
// with identical payloads produce the same pid, which is the
|
||||
// contract the docstring promises. On retry the caller gets back
|
||||
// the same id and the pre-flight idempotency check below short-
|
||||
// circuits to the existing entry instead of re-appending.
|
||||
let mut names_sorted = req.endorsed_names.clone();
|
||||
names_sorted.sort();
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut h = Sha256::new();
|
||||
h.update(ts.as_bytes());
|
||||
h.update(b"|");
|
||||
h.update(req.parent_id.as_bytes());
|
||||
h.update(b"|");
|
||||
h.update(req.operation.as_bytes());
|
||||
h.update(req.parent_id.as_bytes()); h.update(b"|");
|
||||
h.update(req.operation.as_bytes()); h.update(b"|");
|
||||
h.update(req.approach.as_bytes()); h.update(b"|");
|
||||
h.update(req.context.as_bytes()); h.update(b"|");
|
||||
h.update(names_sorted.join(",").as_bytes());
|
||||
let bytes = h.finalize();
|
||||
let pid = format!("pb-rev-{}", bytes.iter().take(8).map(|b| format!("{b:02x}")).collect::<String>());
|
||||
|
||||
// Idempotency short-circuit — if this exact pid already exists in
|
||||
// memory (from a prior successful revise with the same content),
|
||||
// return it directly rather than re-appending or 400ing on the
|
||||
// superseded-parent rejection. Walks the parent's chain via
|
||||
// history() because the parent may itself have been superseded by
|
||||
// our prior successful call.
|
||||
let chain = state.playbook_memory.history(&req.parent_id).await;
|
||||
if let Some(existing) = chain.iter().find(|e| e.playbook_id == pid) {
|
||||
return Ok(Json(serde_json::json!({
|
||||
"outcome": {
|
||||
"parent_id": req.parent_id,
|
||||
"parent_version": existing.version.saturating_sub(1),
|
||||
"new_playbook_id": existing.playbook_id,
|
||||
"new_version": existing.version,
|
||||
"superseded_at": existing.superseded_at.clone().unwrap_or_default(),
|
||||
"idempotent_return": true,
|
||||
},
|
||||
"entries_after": state.playbook_memory.entry_count().await,
|
||||
})));
|
||||
}
|
||||
|
||||
let ts = chrono::Utc::now().to_rfc3339();
|
||||
let new_entry = playbook_memory::PlaybookEntry {
|
||||
playbook_id: pid.clone(),
|
||||
operation: req.operation,
|
||||
|
||||
Reference in New Issue
Block a user