Phase 28-36 body of work

Accumulated since a6f12e2 (Phase 21 Rust port + Phase 27 versioning):

- Phase 36: embed_semaphore on VectorState (permits=1) serializes
  seed embed calls — prevents sidecar socket collisions under
  concurrent /seed stress load
- Phase 31+: run_stress.ts 6-task diverse stress scaffolding;
  run_e2e_rated.ts + orchestrator.ts tightening
- Catalog dedupe cleanup: 16 duplicate manifests removed; canonical
  candidates.parquet (10.5MB -> 76KB) + placements.parquet (1.2MB ->
  11KB) regenerated post-dedupe; fresh manifests for active datasets
- vectord: harness EvalSet refinements (+181), agent portfolio
  rotation + ingest triggers (+158), autotune + rag adjustments
- catalogd/storaged/ingestd/mcp-server: misc tightening
- docs: Phase 28-36 PRD entries + DECISIONS ADR additions;
  control-plane pivot banner added to top of docs/PRD.md (pointing
  at docs/CONTROL_PLANE_PRD.md which lands in next commit)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/catalogd/src/grpc.rs

@@ -54,7 +54,16 @@ impl CatalogService for CatalogGrpc {
         let manifest = self.registry
             .register(req.name, SchemaFingerprint(req.schema_fingerprint), objects)
             .await
-            .map_err(|e| Status::internal(e))?;
+            .map_err(|e| {
+                // Mirror the HTTP side's 409 mapping: schema drift is a
+                // precondition failure, not a server error. Keeps gRPC
+                // and HTTP callers seeing the same diagnostic signal.
+                if e.contains("different schema") {
+                    Status::failed_precondition(e)
+                } else {
+                    Status::internal(e)
+                }
+            })?;
 
         Ok(Response::new(manifest_to_proto(&manifest)))
     }

crates/catalogd/src/registry.rs

@@ -28,6 +28,27 @@ pub struct MigrateBucketsReport {
     pub manifests_persisted: usize,
 }
 
+#[derive(Debug, Clone, Default, serde::Serialize)]
+pub struct DedupeReport {
+    pub groups: usize,
+    pub removed: usize,
+    pub kept: Vec<DedupeKept>,
+    pub errors: Vec<DedupeError>,
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct DedupeKept {
+    pub name: String,
+    pub kept_id: String,
+    pub removed: usize,
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct DedupeError {
+    pub manifest_id: String,
+    pub error: String,
+}
+
 /// Partial metadata update — only set fields are applied.
 #[derive(Debug, Clone, Default, serde::Deserialize)]
 pub struct MetadataUpdate {
@@ -212,13 +233,65 @@ impl Registry {
         Ok(())
     }
 
-    /// Register a new dataset. Persists manifest to storage before updating memory.
+    /// Register a dataset. Idempotent on `name`:
+    ///   - No existing dataset by this name → create new manifest.
+    ///   - Exists with same schema fingerprint → update objects + `updated_at`
+    ///     in place (re-ingest of identical-shape data).
+    ///   - Exists with different schema fingerprint → reject as `Err`
+    ///     (callers map to HTTP 409 / gRPC FAILED_PRECONDITION). Schema
+    ///     evolution is not yet exposed as an HTTP endpoint; callers must
+    ///     register under a new name or delete the existing manifest first.
+    ///
+    /// Concurrency: the write lock is held across the storage write. That
+    /// serializes concurrent registers but is correctness-mandatory —
+    /// dropping the lock across the check→insert sequence creates a TOCTOU
+    /// window where two callers with the same name both see "no existing"
+    /// and both insert, reintroducing the duplicate-manifest bug this
+    /// function exists to prevent. Register is a metadata-only hop on the
+    /// ingest path, so the serialization cost is acceptable. Precedent:
+    /// `update_metadata` also holds the write lock across its I/O.
+    ///
+    /// Legacy state: if multiple manifests with the same name already exist
+    /// (from before this was idempotent), the most recently updated one is
+    /// treated as canonical. Run `dedupe_by_name` to clean up the rest.
     pub async fn register(
         &self,
         name: String,
         schema_fingerprint: SchemaFingerprint,
         objects: Vec<ObjectRef>,
     ) -> Result<DatasetManifest, String> {
+        let mut datasets = self.datasets.write().await;
+
+        let existing = datasets
+            .values()
+            .filter(|d| d.name == name)
+            .max_by_key(|d| d.updated_at)
+            .cloned();
+
+        if let Some(mut manifest) = existing {
+            if manifest.schema_fingerprint != schema_fingerprint {
+                return Err(format!(
+                    "dataset '{}' already exists with a different schema \
+                     (existing fingerprint: {}, new: {}). Schema-evolution \
+                     migration is not yet exposed as an HTTP endpoint; \
+                     register under a new name or delete the existing \
+                     manifest first.",
+                    name, manifest.schema_fingerprint.0, schema_fingerprint.0,
+                ));
+            }
+            manifest.objects = objects;
+            manifest.updated_at = chrono::Utc::now();
+
+            let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
+            let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?;
+            ops::put(&self.store, &manifest_key, json.into()).await?;
+
+            datasets.insert(manifest.id.clone(), manifest.clone());
+
+            tracing::info!("re-registered (idempotent): {} ({})", manifest.name, manifest.id);
+            return Ok(manifest);
+        }
+
         let now = chrono::Utc::now();
         let manifest = DatasetManifest {
             id: DatasetId::new(),
@@ -240,12 +313,10 @@ impl Registry {
             embedding_refresh_policy: None,
         };
 
-        // Write-ahead: persist before in-memory update
         let manifest_key = format!("{MANIFEST_PREFIX}/{}.json", manifest.id);
         let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?;
         ops::put(&self.store, &manifest_key, json.into()).await?;
 
-        let mut datasets = self.datasets.write().await;
         datasets.insert(manifest.id.clone(), manifest.clone());
 
         tracing::info!("registered dataset: {} ({})", manifest.name, manifest.id);
@@ -380,6 +451,72 @@ impl Registry {
         Ok(manifest.clone())
     }
 
+    /// Collapse duplicate manifests that share a `name`. Winner per group:
+    ///   1. Prefer a manifest with a non-null `row_count` (already resynced).
+    ///   2. Break ties by newest `updated_at`.
+    /// Losers are removed from the in-memory registry AND their manifest
+    /// JSON is deleted from object storage. Datasets with a single manifest
+    /// are untouched. Parquet data files are never touched — only catalog
+    /// metadata. See `register` for the prevention side of this fix.
+    ///
+    /// Operator-only. Do not run while ingest is active: a concurrent
+    /// `register` between the snapshot and the delete sweep can create a
+    /// new manifest under a name we just deduped, producing a transient
+    /// count of 2 until the next dedupe run. Safe outside ingest windows.
+    pub async fn dedupe_by_name(&self) -> DedupeReport {
+        let groups: Vec<(String, Vec<DatasetManifest>)> = {
+            let datasets = self.datasets.read().await;
+            let mut by_name: HashMap<String, Vec<DatasetManifest>> = HashMap::new();
+            for m in datasets.values() {
+                by_name.entry(m.name.clone()).or_default().push(m.clone());
+            }
+            by_name.into_iter().filter(|(_, v)| v.len() > 1).collect()
+        };
+
+        let mut report = DedupeReport::default();
+        let mut to_delete: Vec<DatasetId> = Vec::new();
+
+        for (name, mut manifests) in groups {
+            report.groups += 1;
+            manifests.sort_by(|a, b| {
+                b.row_count.is_some().cmp(&a.row_count.is_some())
+                    .then(b.updated_at.cmp(&a.updated_at))
+            });
+            let winner = &manifests[0];
+            report.kept.push(DedupeKept {
+                name: name.clone(),
+                kept_id: winner.id.to_string(),
+                removed: manifests.len() - 1,
+            });
+            for loser in &manifests[1..] {
+                to_delete.push(loser.id.clone());
+            }
+        }
+
+        {
+            let mut datasets = self.datasets.write().await;
+            for id in &to_delete {
+                datasets.remove(id);
+            }
+        }
+        for id in &to_delete {
+            let key = format!("{MANIFEST_PREFIX}/{}.json", id);
+            match ops::delete(&self.store, &key).await {
+                Ok(_) => report.removed += 1,
+                Err(e) => report.errors.push(DedupeError {
+                    manifest_id: id.to_string(),
+                    error: e,
+                }),
+            }
+        }
+
+        tracing::info!(
+            "dedupe: {} groups collapsed, {} manifests removed, {} errors",
+            report.groups, report.removed, report.errors.len(),
+        );
+        report
+    }
+
     /// Resync every dataset that currently has a null row_count.
     /// Returns (successes, failures) where each entry is (name, detail).
     pub async fn resync_missing(&self) -> (Vec<(String, u64)>, Vec<(String, String)>) {
@@ -549,6 +686,46 @@ impl Registry {
         Ok(())
     }
 
+    /// Remove a dataset from the catalog by name. Deletes the manifest
+    /// from both the in-memory registry and object storage.
+    ///
+    /// Scope is metadata-only: the underlying parquet files, vector
+    /// indexes, tombstones, trial journals, and AiViews that reference
+    /// this dataset are NOT touched. Caller is responsible for any
+    /// cascade cleanup. This mirrors how a DROP TABLE in a typical
+    /// warehouse separates "forget about this dataset" from "actually
+    /// reclaim the bytes".
+    ///
+    /// Returns Ok(()) when the dataset existed and was removed; an Err
+    /// with "dataset not found" when no manifest by that name exists.
+    /// Legacy state with duplicate names: all matching manifests are
+    /// removed (effectively a post-hoc dedupe on that name).
+    pub async fn delete_dataset(&self, name: &str) -> Result<usize, String> {
+        let ids_to_remove: Vec<DatasetId> = {
+            let datasets = self.datasets.read().await;
+            datasets.values().filter(|d| d.name == name).map(|d| d.id.clone()).collect()
+        };
+        if ids_to_remove.is_empty() {
+            return Err(format!("dataset not found: {name}"));
+        }
+
+        for id in &ids_to_remove {
+            let key = format!("{MANIFEST_PREFIX}/{}.json", id);
+            if let Err(e) = ops::delete(&self.store, &key).await {
+                // Storage delete failed — log and keep going. The in-memory
+                // remove below is still correct; on restart, the missing
+                // storage object just means nothing to rehydrate.
+                tracing::warn!("delete_dataset '{name}': storage delete of {key} failed: {e}");
+            }
+        }
+        let mut datasets = self.datasets.write().await;
+        for id in &ids_to_remove {
+            datasets.remove(id);
+        }
+        tracing::info!("deleted dataset '{}' ({} manifest(s))", name, ids_to_remove.len());
+        Ok(ids_to_remove.len())
+    }
+
     /// List datasets whose `embedding_stale_since` is set — they need a refresh.
     pub async fn stale_datasets(&self) -> Vec<DatasetManifest> {
         let datasets = self.datasets.read().await;
@@ -579,3 +756,275 @@ impl Registry {
         Ok(manifest.clone())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use object_store::memory::InMemory;
+
+    fn fixture() -> Registry {
+        Registry::new(Arc::new(InMemory::new()))
+    }
+
+    fn fp(s: &str) -> SchemaFingerprint {
+        SchemaFingerprint(s.to_string())
+    }
+
+    fn obj(key: &str) -> ObjectRef {
+        ObjectRef {
+            bucket: "primary".to_string(),
+            key: key.to_string(),
+            size_bytes: 100,
+            created_at: chrono::Utc::now(),
+        }
+    }
+
+    #[tokio::test]
+    async fn fresh_register_creates_new_manifest() {
+        let reg = fixture();
+        let m = reg.register("a".into(), fp("f1"), vec![obj("a.parquet")]).await.unwrap();
+        assert_eq!(m.name, "a");
+        assert_eq!(m.schema_fingerprint, fp("f1"));
+        assert_eq!(reg.list().await.len(), 1);
+        assert_eq!(reg.get_by_name("a").await.unwrap().id, m.id);
+    }
+
+    #[tokio::test]
+    async fn re_register_same_fingerprint_is_idempotent() {
+        let reg = fixture();
+        let first = reg.register("a".into(), fp("f1"), vec![obj("old.parquet")]).await.unwrap();
+        let second = reg.register("a".into(), fp("f1"), vec![obj("new.parquet")]).await.unwrap();
+
+        assert_eq!(first.id, second.id, "same name+fp should reuse ID");
+        assert_eq!(reg.list().await.len(), 1, "no duplicate manifest created");
+        let fetched = reg.get_by_name("a").await.unwrap();
+        assert_eq!(fetched.objects[0].key, "new.parquet", "objects replaced");
+        assert!(fetched.updated_at >= first.updated_at, "updated_at bumped");
+    }
+
+    #[tokio::test]
+    async fn re_register_different_fingerprint_rejects() {
+        let reg = fixture();
+        reg.register("a".into(), fp("f1"), vec![obj("a.parquet")]).await.unwrap();
+        let err = reg.register("a".into(), fp("f2"), vec![obj("a.parquet")]).await.unwrap_err();
+
+        assert!(err.contains("different schema"), "error mentions schema drift: {err}");
+        assert_eq!(reg.list().await.len(), 1, "failed register did not mutate state");
+        assert_eq!(reg.get_by_name("a").await.unwrap().schema_fingerprint, fp("f1"));
+    }
+
+    #[tokio::test]
+    async fn delete_dataset_removes_manifest_and_in_memory_entry() {
+        let reg = fixture();
+        reg.register("to_delete".into(), fp("f1"), vec![obj("d.parquet")]).await.unwrap();
+        reg.register("keepme".into(),    fp("f2"), vec![obj("k.parquet")]).await.unwrap();
+        assert_eq!(reg.list().await.len(), 2);
+
+        let removed = reg.delete_dataset("to_delete").await.unwrap();
+        assert_eq!(removed, 1, "one manifest removed");
+        assert!(reg.get_by_name("to_delete").await.is_none());
+        assert!(reg.get_by_name("keepme").await.is_some(), "unrelated dataset untouched");
+        assert_eq!(reg.list().await.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn delete_dataset_404s_on_unknown_name() {
+        let reg = fixture();
+        let err = reg.delete_dataset("nope").await.unwrap_err();
+        assert!(err.starts_with("dataset not found"), "error wording: {err}");
+    }
+
+    #[tokio::test]
+    async fn delete_dataset_removes_all_dupes_with_matching_name() {
+        // Legacy zombies: direct insert so we can seed duplicates,
+        // mirroring pre-idempotent-register state. delete_dataset should
+        // sweep every manifest sharing that name in a single call.
+        let reg = fixture();
+        let now = chrono::Utc::now();
+        for _ in 0..3 {
+            let m = DatasetManifest {
+                id: DatasetId::new(),
+                name: "zombie".into(),
+                schema_fingerprint: fp("fp"),
+                objects: vec![obj("z.parquet")],
+                created_at: now, updated_at: now,
+                row_count: None, description: String::new(), owner: String::new(),
+                sensitivity: None, columns: vec![], lineage: None, freshness: None,
+                tags: vec![], last_embedded_at: None, embedding_stale_since: None,
+                embedding_refresh_policy: None,
+            };
+            reg.datasets.write().await.insert(m.id.clone(), m);
+        }
+        assert_eq!(reg.list().await.len(), 3);
+
+        let removed = reg.delete_dataset("zombie").await.unwrap();
+        assert_eq!(removed, 3, "all three manifests removed in one sweep");
+        assert_eq!(reg.list().await.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn dedupe_no_duplicates_is_noop() {
+        let reg = fixture();
+        reg.register("a".into(), fp("f1"), vec![obj("a.parquet")]).await.unwrap();
+        reg.register("b".into(), fp("f2"), vec![obj("b.parquet")]).await.unwrap();
+
+        let report = reg.dedupe_by_name().await;
+        assert_eq!(report.groups, 0);
+        assert_eq!(report.removed, 0);
+        assert_eq!(reg.list().await.len(), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn concurrent_register_stress_many_workers_one_manifest() {
+        // Stress variant: fire 32 concurrent registers for the same new
+        // name across 8 worker threads. Single-manifest invariant must
+        // hold regardless of how the scheduler interleaves them. This
+        // catches race regressions the 2-call variant below might miss
+        // under a forgiving scheduler.
+        let reg = fixture();
+        let mut handles = Vec::new();
+        for i in 0..32 {
+            let r = reg.clone();
+            handles.push(tokio::spawn(async move {
+                r.register(
+                    "stress".into(),
+                    fp("stress-fp"),
+                    vec![obj(&format!("{i}.parquet"))],
+                ).await
+            }));
+        }
+        for h in handles {
+            h.await.unwrap().expect("each register succeeds (idempotent)");
+        }
+        let all = reg.list().await;
+        let stress_manifests: Vec<_> = all.iter().filter(|d| d.name == "stress").collect();
+        assert_eq!(stress_manifests.len(), 1,
+            "32 concurrent registers produced {} manifests for 'stress' — expected 1",
+            stress_manifests.len());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn concurrent_register_same_new_name_collapses_to_one_manifest() {
+        // Pins the TOCTOU fix: without the write-lock held across the
+        // check→insert sequence, two parallel registers of the same
+        // previously-unknown name would both see "no existing" and both
+        // insert. Multi-thread flavor is required to actually exercise
+        // the race — under single-threaded tokio the scheduler doesn't
+        // preempt between awaits on the same task's critical section.
+        // Under the current impl, whichever acquires the write lock
+        // first creates the manifest; the second observes it and takes
+        // the idempotent-update branch.
+        let reg = fixture();
+        let a = reg.clone();
+        let b = reg.clone();
+
+        let (ra, rb) = tokio::join!(
+            async move {
+                a.register("race".into(), fp("shared-fp"), vec![obj("a.parquet")]).await
+            },
+            async move {
+                b.register("race".into(), fp("shared-fp"), vec![obj("b.parquet")]).await
+            },
+        );
+
+        let m_a = ra.expect("first register succeeds");
+        let m_b = rb.expect("second register succeeds (same fp = idempotent)");
+        assert_eq!(m_a.id, m_b.id, "both calls resolve to a single DatasetId");
+        assert_eq!(reg.list().await.len(), 1, "no duplicate manifest created");
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn concurrent_register_same_name_different_fp_one_wins_one_rejects() {
+        // Reverse case: concurrent registers with conflicting fingerprints.
+        // Deterministic outcome: one call's fingerprint gets persisted as
+        // canonical, the other is rejected on seeing the now-established
+        // fingerprint. Which one wins is scheduler-dependent — we only
+        // assert exactly one success and exactly one rejection.
+        let reg = fixture();
+        let a = reg.clone();
+        let b = reg.clone();
+
+        let (ra, rb) = tokio::join!(
+            async move { a.register("race".into(), fp("fp1"), vec![obj("a.parquet")]).await },
+            async move { b.register("race".into(), fp("fp2"), vec![obj("b.parquet")]).await },
+        );
+
+        let successes = [&ra, &rb].iter().filter(|r| r.is_ok()).count();
+        let rejections = [&ra, &rb].iter().filter(|r| r.is_err()).count();
+        assert_eq!(successes, 1, "exactly one register wins");
+        assert_eq!(rejections, 1, "the other is rejected on fingerprint");
+        let rejection_msg = [&ra, &rb].iter()
+            .find_map(|r| r.as_ref().err())
+            .unwrap();
+        assert!(rejection_msg.contains("different schema"));
+        assert_eq!(reg.list().await.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn dedupe_collapses_dupes_preferring_non_null_row_count() {
+        let reg = fixture();
+
+        // Simulate legacy pre-idempotent-register state by inserting
+        // manifests with the same `name` directly. Mirrors what happened
+        // with the 308× successful_playbooks duplication.
+        let now = chrono::Utc::now();
+        let old_with_rows = DatasetManifest {
+            id: DatasetId::new(),
+            name: "dupes".into(),
+            schema_fingerprint: fp("f1"),
+            objects: vec![obj("dupes.parquet")],
+            created_at: now - chrono::Duration::hours(2),
+            updated_at: now - chrono::Duration::hours(2),
+            row_count: Some(42),
+            description: String::new(),
+            owner: String::new(),
+            sensitivity: None,
+            columns: vec![],
+            lineage: None,
+            freshness: None,
+            tags: vec![],
+            last_embedded_at: None,
+            embedding_stale_since: None,
+            embedding_refresh_policy: None,
+        };
+        let newer_no_rows = DatasetManifest {
+            id: DatasetId::new(),
+            updated_at: now, // newer, but null row_count
+            row_count: None,
+            ..old_with_rows.clone()
+        };
+        let oldest = DatasetManifest {
+            id: DatasetId::new(),
+            updated_at: now - chrono::Duration::hours(5),
+            row_count: None,
+            ..old_with_rows.clone()
+        };
+
+        let kept_id = old_with_rows.id.clone();
+        let loser_a = newer_no_rows.id.clone();
+        let loser_b = oldest.id.clone();
+
+        {
+            let mut d = reg.datasets.write().await;
+            d.insert(old_with_rows.id.clone(), old_with_rows);
+            d.insert(newer_no_rows.id.clone(), newer_no_rows);
+            d.insert(oldest.id.clone(), oldest);
+        }
+        assert_eq!(reg.list().await.len(), 3);
+
+        let report = reg.dedupe_by_name().await;
+        assert_eq!(report.groups, 1);
+        assert_eq!(report.removed, 2);
+        assert_eq!(report.errors.len(), 0);
+        assert_eq!(report.kept.len(), 1);
+        assert_eq!(report.kept[0].name, "dupes");
+        assert_eq!(report.kept[0].kept_id, kept_id.to_string(),
+            "should keep the manifest with non-null row_count");
+
+        let remaining = reg.list().await;
+        assert_eq!(remaining.len(), 1);
+        assert_eq!(remaining[0].id, kept_id);
+        assert!(reg.get(&loser_a).await.is_none());
+        assert!(reg.get(&loser_b).await.is_none());
+    }
+}

crates/catalogd/src/service.rs

@@ -17,11 +17,12 @@ pub fn router(registry: Registry) -> Router {
         .route("/datasets", post(create_dataset))
         .route("/datasets", get(list_datasets))
         .route("/datasets/{id}", get(get_dataset))
-        .route("/datasets/by-name/{name}", get(get_dataset_by_name))
+        .route("/datasets/by-name/{name}", get(get_dataset_by_name).delete(delete_dataset_by_name))
         .route("/datasets/by-name/{name}/metadata", post(update_metadata))
         .route("/datasets/by-name/{name}/resync", post(resync_dataset))
         .route("/resync-missing", post(resync_all_missing))
         .route("/migrate-buckets", post(migrate_buckets))
+        .route("/dedupe", post(dedupe_by_name))
         // Phase D: AI-safe views
         .route("/views", post(create_view).get(list_views))
         .route("/views/{name}", get(get_view).delete(delete_view))
@@ -121,7 +122,14 @@ async fn create_dataset(
             let resp = DatasetResponse::from(&manifest);
             Ok((StatusCode::CREATED, Json(resp)))
         }
-        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+        Err(e) => {
+            let status = if e.contains("different schema") {
+                StatusCode::CONFLICT
+            } else {
+                StatusCode::INTERNAL_SERVER_ERROR
+            };
+            Err((status, e))
+        }
     }
 }
 
@@ -153,6 +161,25 @@ async fn get_dataset_by_name(
     }
 }
 
+/// Remove a dataset manifest by name. Metadata only — parquet files,
+/// vector indexes, and tombstones are NOT cascade-deleted. See
+/// `Registry::delete_dataset` for the full scope.
+async fn delete_dataset_by_name(
+    State(registry): State<Registry>,
+    Path(name): Path<String>,
+) -> impl IntoResponse {
+    match registry.delete_dataset(&name).await {
+        Ok(removed) => {
+            let body = serde_json::json!({ "name": name, "manifests_removed": removed });
+            Ok(Json(body))
+        }
+        Err(e) if e.starts_with("dataset not found") => {
+            Err((StatusCode::NOT_FOUND, e))
+        }
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
 async fn update_metadata(
     State(registry): State<Registry>,
     Path(name): Path<String>,
@@ -214,6 +241,12 @@ async fn migrate_buckets(State(registry): State<Registry>) -> impl IntoResponse
     }
 }
 
+/// Collapse duplicate manifests by `name` — see `Registry::dedupe_by_name`.
+/// Safe to run repeatedly; single-manifest datasets are untouched.
+async fn dedupe_by_name(State(registry): State<Registry>) -> impl IntoResponse {
+    Json(registry.dedupe_by_name().await)
+}
+
 // --- Phase D: AI-safe views ---
 
 #[derive(Deserialize)]

crates/gateway/src/main.rs

@@ -86,6 +86,11 @@ async fn main() {
     // primary. Journal / registry resolve per-call via the bucket registry.
     let tj = vectord::trial::TrialJournal::new(bucket_registry.clone(), index_reg.clone());
     let pr = vectord::promotion::PromotionRegistry::new(bucket_registry.clone(), index_reg.clone());
+    let hs = vectord::harness::HarnessStore::new(bucket_registry.clone(), index_reg.clone());
+    // Phase 19: playbook memory. Load cached state; empty-on-miss is fine,
+    // operators call POST /vectors/playbook_memory/rebuild to populate.
+    let pbm = vectord::playbook_memory::PlaybookMemory::new(store.clone());
+    let _ = pbm.load_from_storage().await;
 
     // Phase 16.2: spawn the autotune agent. When config.agent.enabled=false
     // this returns a handle that drops triggers silently — no surprise load.
@@ -107,6 +112,7 @@ async fn main() {
             embedding_cache: emb_cache.clone(),
             trial_journal: tj.clone(),
             promotion_registry: pr.clone(),
+            harness_store: hs.clone(),
         },
     );
 
@@ -155,6 +161,7 @@ async fn main() {
             hnsw_store: hnsw,
             embedding_cache: emb_cache,
             trial_journal: tj,
+            harness_store: hs,
             catalog: registry.clone(),
             promotion_registry: pr,
             agent_handle,
@@ -163,6 +170,8 @@ async fn main() {
             lance: vectord::lance_backend::LanceRegistry::new(
                 bucket_registry.clone(), index_reg.clone(),
             ),
+            playbook_memory: pbm,
+            embed_semaphore: std::sync::Arc::new(tokio::sync::Semaphore::new(1)),
         }))
         .nest("/workspaces", queryd::workspace_service::router(workspace_mgr))
         .nest("/journal", journald::service::router(journal))

crates/gateway/src/tools/registry.rs

@@ -103,7 +103,7 @@ impl ToolRegistry {
                     ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
                 ],
                 returns: "List of candidates with id, name, phone, email, skills, experience".into(),
-                sql_template: "SELECT candidate_id, first_name, last_name, phone, email, city, state, zip, vertical, skills, years_experience FROM candidates WHERE 1=1 {skills_filter} {city_filter} {state_filter} {years_filter} {status_filter} ORDER BY years_experience DESC LIMIT {limit}".into(),
+                sql_template: "SELECT candidate_id, first_name, last_name, phone, email, city, state, skills, years_experience, status FROM candidates WHERE 1=1 {skills_filter} {city_filter} {state_filter} {years_filter} {status_filter} ORDER BY years_experience DESC LIMIT {limit}".into(),
                 category: "candidates".into(),
             },
             ToolDef {

crates/ingestd/Cargo.toml

@@ -24,3 +24,4 @@ object_store = { workspace = true }
 tokio-postgres = { workspace = true }
 mysql_async = { workspace = true }
 uuid = { workspace = true }
+croner = "3"

crates/ingestd/src/schedule.rs

@@ -17,8 +17,8 @@
 //! shouldn't pile up runs). Different schedules run concurrently.
 //!
 //! What's deliberately not in scope here:
-//! - Cron expressions (the trigger enum has the variant but parsing is
-//!   stubbed). Intervals cover 90% of operational scheduling; cron is
+//! - Sub-minute cron precision. 5-field Unix cron is supported; seconds
+//!   are pinned to 0. Intervals cover sub-minute cases; cron is
 //!   easy to bolt on later.
 //! - Backoff / retry policies. A failed run records the failure and
 //!   schedules `next_run_at` as if it succeeded — no exponential backoff.
@@ -85,8 +85,12 @@ impl ScheduleKind {
 pub enum ScheduleTrigger {
     /// Run every N seconds. Time anchor = compute_next_run_at decision.
     Interval { secs: u64 },
-    /// Cron expression — parsing not implemented yet. Defining the
-    /// variant now so the JSON shape is forward-compatible.
+    /// 5-field Unix cron expression: "min hour dom month dow".
+    /// Seconds are implicitly 0 (minute-granularity scheduling).
+    /// Examples: "15 14 * * *" = 14:15 UTC daily, "0 */6 * * *" = every
+    /// 6 hours on the hour, "0 9 * * 1-5" = 09:00 UTC weekdays.
+    /// Timezone: UTC always — the scheduler's clock is UTC and we don't
+    /// carry a per-schedule timezone field.
     Cron { expr: String },
 }
 
@@ -162,22 +166,55 @@ impl ScheduleDef {
 //      Parse the expression, find the next match after `now`. Most
 //    expressive but pulls in a cron parser dep.
 //
-// Cron is currently stubbed — falling back to "now + 1 hour" so a
-// mistakenly-configured Cron schedule doesn't melt the GPU.
 pub fn compute_next_run_at(
     trigger: &ScheduleTrigger,
     completed_at: DateTime<Utc>,
     _previous_next: DateTime<Utc>,
 ) -> DateTime<Utc> {
-    // TODO(J): pick a strategy. Starter = since-last-completion.
     match trigger {
         ScheduleTrigger::Interval { secs } => {
             completed_at + Duration::seconds(*secs as i64)
         }
-        ScheduleTrigger::Cron { expr: _ } => {
-            // Cron parsing not implemented — fall back to a safe 1h
-            // window so a bad config can't fire-loop the system.
-            completed_at + Duration::hours(1)
+        ScheduleTrigger::Cron { expr } => {
+            next_cron_fire(expr, completed_at)
+                // Creation-time validation (see `validate_trigger`) rejects
+                // unparseable expressions, so this fallback is only reached
+                // if the schedule JSON was hand-edited on disk. One-hour
+                // window keeps a bad expr from fire-looping the system.
+                .unwrap_or(completed_at + Duration::hours(1))
+        }
+    }
+}
+
+/// Parse a Vixie/POSIX cron expression and return the next fire time
+/// strictly after `after`. Accepts 5-field (`min hour dom month dow`) and
+/// 6-field (with leading `sec`) forms natively — croner handles both.
+/// Day-of-week follows Unix convention: 0=Sun, 1=Mon, …, 6=Sat (7=Sun as
+/// synonym). Returns None if the expression fails to parse or produces no
+/// upcoming match (which can happen for year-bounded patterns in the past).
+fn next_cron_fire(expr: &str, after: DateTime<Utc>) -> Option<DateTime<Utc>> {
+    use std::str::FromStr;
+    let cron = croner::Cron::from_str(expr).ok()?;
+    cron.find_next_occurrence(&after, false).ok()
+}
+
+/// Reject a ScheduleTrigger whose content can't be interpreted — used by
+/// the HTTP handlers to fail fast at create/patch time rather than falling
+/// back silently at fire time. Interval triggers get a sanity gate
+/// (no zero secs); cron triggers get a full parse through croner.
+pub fn validate_trigger(trigger: &ScheduleTrigger) -> Result<(), String> {
+    match trigger {
+        ScheduleTrigger::Interval { secs } => {
+            if *secs == 0 {
+                return Err("interval secs must be > 0".into());
+            }
+            Ok(())
+        }
+        ScheduleTrigger::Cron { expr } => {
+            use std::str::FromStr;
+            croner::Cron::from_str(expr)
+                .map_err(|e| format!("invalid cron expression '{expr}': {e}"))?;
+            Ok(())
         }
     }
 }
@@ -551,6 +588,7 @@ fn redact_dsn(dsn: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use chrono::Timelike;
 
     fn mk(secs: u64) -> ScheduleDef {
         ScheduleDef {
@@ -600,12 +638,73 @@ mod tests {
     }
 
     #[test]
-    fn cron_falls_back_to_one_hour() {
+    fn cron_every_minute_fires_within_sixty_seconds() {
         let trig = ScheduleTrigger::Cron { expr: "* * * * *".into() };
         let now = Utc::now();
         let next = compute_next_run_at(&trig, now, now);
         let delta = next - now;
-        assert_eq!(delta.num_seconds(), 3600);
+        assert!(delta.num_seconds() > 0 && delta.num_seconds() <= 60,
+            "expected next fire within 60s, got {}s", delta.num_seconds());
+    }
+
+    #[test]
+    fn cron_daily_at_1415_utc_is_within_24h() {
+        // 14:15 UTC daily — whether we're before or after 14:15 today, the
+        // next fire is at most 24h out.
+        let trig = ScheduleTrigger::Cron { expr: "15 14 * * *".into() };
+        let now: DateTime<Utc> = "2026-04-20T10:00:00Z".parse().unwrap();
+        let next = compute_next_run_at(&trig, now, now);
+        assert!(next > now);
+        assert!((next - now).num_hours() <= 24);
+        // And the minute-of-hour is 15.
+        assert_eq!(next.minute(), 15);
+        assert_eq!(next.hour(), 14);
+    }
+
+    #[test]
+    fn cron_weekday_skips_weekend() {
+        // 09:00 UTC weekdays only. On a Saturday, next fire is Monday.
+        let trig = ScheduleTrigger::Cron { expr: "0 9 * * 1-5".into() };
+        // 2026-04-18 is a Saturday.
+        let sat: DateTime<Utc> = "2026-04-18T10:00:00Z".parse().unwrap();
+        let next = compute_next_run_at(&trig, sat, sat);
+        // Monday 2026-04-20 at 09:00 UTC.
+        assert_eq!(next, "2026-04-20T09:00:00Z".parse::<DateTime<Utc>>().unwrap());
+    }
+
+    #[test]
+    fn cron_six_field_seconds_granularity() {
+        // 6-field (seconds included): fire every minute at 30s past the minute.
+        let trig = ScheduleTrigger::Cron { expr: "30 * * * * *".into() };
+        let now = Utc::now();
+        let next = compute_next_run_at(&trig, now, now);
+        assert_eq!(next.second(), 30);
+    }
+
+    #[test]
+    fn validate_rejects_bad_cron() {
+        let bad = ScheduleTrigger::Cron { expr: "not a cron".into() };
+        let err = validate_trigger(&bad).unwrap_err();
+        assert!(err.contains("cron"), "error should mention cron, got: {err}");
+    }
+
+    #[test]
+    fn validate_rejects_wrong_field_count() {
+        // 4 fields — neither 5 nor 6.
+        let bad = ScheduleTrigger::Cron { expr: "* * * *".into() };
+        assert!(validate_trigger(&bad).is_err());
+    }
+
+    #[test]
+    fn validate_rejects_zero_interval() {
+        let bad = ScheduleTrigger::Interval { secs: 0 };
+        assert!(validate_trigger(&bad).is_err());
+    }
+
+    #[test]
+    fn validate_accepts_good_cron() {
+        let good = ScheduleTrigger::Cron { expr: "0 */6 * * *".into() };
+        assert!(validate_trigger(&good).is_ok());
     }
 
     #[test]

crates/ingestd/src/service.rs

@@ -510,6 +510,9 @@ async fn create_schedule(
     if state.schedules.get(&req.id).await.is_some() {
         return Err((StatusCode::CONFLICT, format!("schedule '{}' already exists", req.id)));
     }
+    if let Err(e) = schedule::validate_trigger(&req.trigger) {
+        return Err((StatusCode::BAD_REQUEST, e));
+    }
     let now = chrono::Utc::now();
     let def = schedule::ScheduleDef {
         id: req.id,
@@ -564,6 +567,11 @@ async fn patch_schedule(
     let Some(mut def) = state.schedules.get(&id).await else {
         return Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found")));
     };
+    if let Some(t) = &req.trigger {
+        if let Err(e) = schedule::validate_trigger(t) {
+            return Err((StatusCode::BAD_REQUEST, e));
+        }
+    }
     if let Some(e) = req.enabled { def.enabled = e; }
     if let Some(t) = req.trigger { def.trigger = t; }
     match state.schedules.put(def).await {

crates/storaged/src/append_log.rs

@@ -18,7 +18,7 @@
 ///   to small JSONL events.
 ///
 /// Storage layout:
-/// ```
+/// ```text
 /// {prefix}/
 ///   batch_0001776319628000123.jsonl
 ///   batch_0001776319745987654.jsonl
@@ -221,15 +221,15 @@ pub struct CompactStats {
     pub new_key: Option<String>,
 }
 
-// Proactively flush on drop, best-effort.
-// We can't `.await` in Drop; we spawn the flush on the tokio runtime if one
-// is available. If the runtime is already shutting down the flush is lost —
-// acceptable for these observability journals, which are hints not records.
+// Log unflushed-buffer size on drop. We can't `.await` from a sync `Drop`,
+// so a real flush isn't possible here — callers are responsible for calling
+// `.flush()` before dropping if durability matters. These journals are
+// observability hints; a few lost buffered events at shutdown are
+// acceptable per ADR-018.
 impl Drop for AppendLog {
     fn drop(&mut self) {
         let buf_len = self.buffer.try_lock().map(|b| b.len()).unwrap_or(0);
         if buf_len == 0 { return; }
-        // Can't spawn from sync Drop on every runtime shape; log + move on.
         tracing::debug!(
             "append_log '{}' dropping with {} unflushed events",
             self.prefix, buf_len,
@@ -237,3 +237,94 @@ impl Drop for AppendLog {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use object_store::memory::InMemory;
+
+    fn mk(threshold: usize) -> AppendLog {
+        AppendLog::new(Arc::new(InMemory::new()), "prefix")
+            .with_flush_threshold(threshold)
+    }
+
+    #[tokio::test]
+    async fn append_stays_buffered_below_threshold() {
+        let log = mk(5);
+        log.append(b"one".to_vec()).await.unwrap();
+        log.append(b"two".to_vec()).await.unwrap();
+        assert_eq!(log.file_count().await.unwrap(), 0, "no files until threshold");
+        let all = log.read_all().await.unwrap();
+        assert_eq!(all, vec![b"one".to_vec(), b"two".to_vec()],
+            "read_all surfaces unflushed buffer");
+    }
+
+    #[tokio::test]
+    async fn append_auto_flushes_on_threshold() {
+        let log = mk(3);
+        for i in 0..3 {
+            log.append(format!("evt{i}").into_bytes()).await.unwrap();
+        }
+        assert_eq!(log.file_count().await.unwrap(), 1, "threshold triggered one flush");
+
+        // A fourth append stays buffered until the next threshold.
+        log.append(b"evt3".to_vec()).await.unwrap();
+        assert_eq!(log.file_count().await.unwrap(), 1, "below threshold again");
+    }
+
+    #[tokio::test]
+    async fn flush_empty_is_noop() {
+        let log = mk(32);
+        log.flush().await.unwrap();
+        log.flush().await.unwrap();
+        assert_eq!(log.file_count().await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn read_all_orders_events_across_flushes() {
+        let log = mk(1); // flush-on-every-append
+        for i in 0..5 {
+            log.append(format!("e{i}").into_bytes()).await.unwrap();
+            // Spread writes out so timestamps sort strictly.
+            tokio::time::sleep(std::time::Duration::from_millis(2)).await;
+        }
+        let all = log.read_all().await.unwrap();
+        let strs: Vec<String> = all.into_iter()
+            .map(|v| String::from_utf8(v).unwrap())
+            .collect();
+        assert_eq!(strs, vec!["e0", "e1", "e2", "e3", "e4"],
+            "lex sort of batch keys == chronological event order");
+    }
+
+    #[tokio::test]
+    async fn compact_merges_multiple_files_into_one() {
+        let log = mk(1); // force file-per-append
+        for i in 0..4 {
+            log.append(format!("e{i}").into_bytes()).await.unwrap();
+            tokio::time::sleep(std::time::Duration::from_millis(2)).await;
+        }
+        assert_eq!(log.file_count().await.unwrap(), 4);
+
+        let stats = log.compact().await.unwrap();
+        assert_eq!(stats.merged_files, 4);
+        assert_eq!(stats.events, 4);
+        assert!(stats.new_key.is_some());
+
+        assert_eq!(log.file_count().await.unwrap(), 1, "originals deleted, 1 survivor");
+        let all = log.read_all().await.unwrap();
+        assert_eq!(all.len(), 4, "no events lost in compaction");
+    }
+
+    #[tokio::test]
+    async fn compact_with_single_file_is_noop() {
+        let log = mk(1);
+        log.append(b"only".to_vec()).await.unwrap();
+        assert_eq!(log.file_count().await.unwrap(), 1);
+
+        let stats = log.compact().await.unwrap();
+        assert_eq!(stats.merged_files, 1);
+        assert_eq!(stats.events, 0, "nothing to consolidate");
+        assert!(stats.new_key.is_none(), "no new file written");
+        assert_eq!(log.file_count().await.unwrap(), 1, "original untouched");
+    }
+}
+

crates/storaged/src/error_journal.rs

@@ -198,3 +198,92 @@ impl ErrorJournal {
         self.log.file_count().await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use object_store::memory::InMemory;
+
+    fn mk() -> ErrorJournal {
+        ErrorJournal::new(Arc::new(InMemory::new()))
+    }
+
+    #[tokio::test]
+    async fn append_updates_ring_immediately() {
+        let j = mk();
+        j.append(BucketErrorEvent::new_read("tenant_a", "k1", "timeout")).await;
+        j.append(BucketErrorEvent::new_write("tenant_b", "k2", "denied")).await;
+
+        let recent = j.recent(10).await;
+        assert_eq!(recent.len(), 2);
+        assert_eq!(recent[0].target, "tenant_a");
+        assert_eq!(recent[1].target, "tenant_b");
+    }
+
+    #[tokio::test]
+    async fn load_recent_rehydrates_ring_from_disk() {
+        // Use a shared InMemory store so the second journal reads the first
+        // journal's writes — mirrors gateway restart semantics.
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        {
+            let j1 = ErrorJournal {
+                log: Arc::new(AppendLog::new(store.clone(), JOURNAL_PREFIX).with_flush_threshold(1)),
+                ring: Arc::new(RwLock::new(VecDeque::with_capacity(RING_CAPACITY))),
+            };
+            j1.append(BucketErrorEvent::new_read("t", "k", "err")).await;
+            j1.flush().await.unwrap();
+        }
+
+        let j2 = ErrorJournal {
+            log: Arc::new(AppendLog::new(store.clone(), JOURNAL_PREFIX)),
+            ring: Arc::new(RwLock::new(VecDeque::with_capacity(RING_CAPACITY))),
+        };
+        assert_eq!(j2.recent(10).await.len(), 0, "fresh ring is empty");
+
+        let loaded = j2.load_recent().await.unwrap();
+        assert_eq!(loaded, 1);
+        assert_eq!(j2.recent(10).await[0].target, "t");
+    }
+
+    #[tokio::test]
+    async fn mark_rescued_updates_most_recent_match() {
+        let j = mk();
+        j.append(BucketErrorEvent::new_read("t1", "k", "err")).await;
+        j.append(BucketErrorEvent::new_read("t2", "k", "err")).await;
+        j.append(BucketErrorEvent::new_read("t1", "k", "err")).await;
+
+        j.mark_rescued_last("t1", "k").await;
+
+        let recent = j.recent(10).await;
+        assert!(!recent[0].rescued, "older t1 event not touched");
+        assert!(!recent[1].rescued, "t2 not touched");
+        assert!(recent[2].rescued, "most recent t1 was rescued");
+    }
+
+    #[tokio::test]
+    async fn health_flags_buckets_with_three_or_more_errors() {
+        let j = mk();
+        for _ in 0..3 { j.append(BucketErrorEvent::new_read("broken", "k", "e")).await; }
+        j.append(BucketErrorEvent::new_read("flaky", "k", "e")).await;
+
+        let h = j.health(60).await;
+        assert_eq!(h.total_errors, 4);
+        assert_eq!(h.per_bucket.get("broken"), Some(&3));
+        assert_eq!(h.per_bucket.get("flaky"), Some(&1));
+        assert!(h.unhealthy_buckets.contains(&"broken".to_string()));
+        assert!(!h.unhealthy_buckets.contains(&"flaky".to_string()));
+    }
+
+    #[tokio::test]
+    async fn filter_by_bucket_returns_chronological_order() {
+        let j = mk();
+        j.append(BucketErrorEvent::new_read("a", "k1", "e")).await;
+        j.append(BucketErrorEvent::new_read("b", "k2", "e")).await;
+        j.append(BucketErrorEvent::new_read("a", "k3", "e")).await;
+
+        let only_a = j.filter(Some("a"), None, 10).await;
+        assert_eq!(only_a.len(), 2);
+        assert_eq!(only_a[0].key, "k1", "oldest first");
+        assert_eq!(only_a[1].key, "k3");
+    }
+}

crates/vectord-lance/src/lib.rs

@@ -357,7 +357,25 @@ impl LanceVectorStore {
     /// Search for top_k nearest neighbors of `query`. Uses the IVF_PQ
     /// index if one exists; otherwise does a full scan (slow but
     /// correct — useful during development before index build).
-    pub async fn search(&self, query: &[f32], top_k: usize) -> Result<Vec<Hit>, String> {
+    ///
+    /// `nprobes` tells Lance how many IVF partitions to probe per query.
+    /// Lance's built-in default is **1**, which caps recall well below
+    /// what the index is actually capable of. Passing `None` keeps that
+    /// default (only sensible when latency trumps recall); for real
+    /// workloads set `nprobes` to 5–10% of `num_partitions` (e.g. 20–30
+    /// on a 316-partition index).
+    ///
+    /// `refine_factor` re-ranks the PQ-approximate top-k by computing
+    /// exact distances on `top_k * refine_factor` candidates, then
+    /// trimming to `top_k`. Cheap way to buy back recall lost to product
+    /// quantization. `None` skips the re-rank pass.
+    pub async fn search(
+        &self,
+        query: &[f32],
+        top_k: usize,
+        nprobes: Option<usize>,
+        refine_factor: Option<u32>,
+    ) -> Result<Vec<Hit>, String> {
         use lance::dataset::Dataset;
 
         let dataset = Dataset::open(&self.path).await.map_err(e)?;
@@ -365,6 +383,8 @@ impl LanceVectorStore {
 
         let mut scanner = dataset.scan();
         scanner.nearest("vector", &qarr, top_k as usize).map_err(e)?;
+        if let Some(n) = nprobes { scanner.nprobes(n); }
+        if let Some(f) = refine_factor { scanner.refine(f); }
         scanner.project(&["doc_id", "chunk_text"]).map_err(e)?;
 
         let mut stream = scanner.try_into_stream().await.map_err(e)?;

crates/vectord/Cargo.toml

@@ -9,6 +9,7 @@ storaged = { path = "../storaged" }
 aibridge = { path = "../aibridge" }
 catalogd = { path = "../catalogd" }
 queryd = { path = "../queryd" }
+futures = { workspace = true }
 # ADR-019 firewall — vectord-lance owns its own Arrow 57 / Lance 4 deps.
 # Public API uses only std types so no version conflict propagates here.
 vectord-lance = { path = "../vectord-lance" }
@@ -24,3 +25,4 @@ arrow = { workspace = true }
 chrono = { workspace = true }
 instant-distance = { workspace = true }
 uuid = { workspace = true }
+sha2 = { workspace = true }

crates/vectord/src/agent.rs

@@ -204,6 +204,7 @@ pub struct AgentDeps {
     pub embedding_cache: EmbeddingCache,
     pub trial_journal: TrialJournal,
     pub promotion_registry: PromotionRegistry,
+    pub harness_store: crate::harness::HarnessStore,
 }
 
 // -------- Spawn --------
@@ -354,12 +355,11 @@ async fn run_one_cycle(
     // Read history.
     let history = deps.trial_journal.list(&event.index_name).await
         .map_err(|e| format!("read journal: {e}"))?;
-    if history.is_empty() {
-        return Err(format!(
-            "no trials yet for '{}' — seed with at least one POST /hnsw/trial first",
-            event.index_name,
-        ));
-    }
+
+    // Bootstrap mode: first ever visit to this index. Auto-generate a
+    // harness (if missing) and seed with the default config — the Phase 15
+    // known-good ec=80/es=30. Subsequent visits go through the proposer.
+    let is_bootstrap = history.is_empty();
 
     // Current champion (if any) is the promoted config.
     let champion = deps.promotion_registry.get_current(&event.index_name).await;
@@ -367,24 +367,26 @@ async fn run_one_cycle(
         history.iter().find(|t| t.id == p.trial_id).cloned()
     });
 
-    // Propose the next config.
-    let Some(next_config) = propose_next_config(&history, champion_trial.as_ref()) else {
-        return Err("proposer returned None — search space exhausted".into());
+    let (next_config, harness_name) = if is_bootstrap {
+        let name = ensure_auto_harness(&event.index_name, deps).await?;
+        (HnswConfig::default(), name)
+    } else {
+        let Some(cfg) = propose_next_config(&history, champion_trial.as_ref()) else {
+            return Err("proposer returned None — search space exhausted".into());
+        };
+        // Validate bounds defensively.
+        if !(10..=400).contains(&cfg.ef_construction) {
+            return Err(format!("proposed ef_construction={} out of bounds", cfg.ef_construction));
+        }
+        if !(10..=200).contains(&cfg.ef_search) {
+            return Err(format!("proposed ef_search={} out of bounds", cfg.ef_search));
+        }
+        // A future refinement: remember per-index "canonical harness" on
+        // the index metadata. For now: latest wins.
+        let hname = history.last().unwrap().eval_set.clone();
+        (cfg, hname)
     };
-
-    // Validate bounds defensively.
-    if !(10..=400).contains(&next_config.ef_construction) {
-        return Err(format!("proposed ef_construction={} out of bounds", next_config.ef_construction));
-    }
-    if !(10..=200).contains(&next_config.ef_search) {
-        return Err(format!("proposed ef_search={} out of bounds", next_config.ef_search));
-    }
-
-    // Need a harness to measure. Use the most recent one in history.
-    // (A future refinement: remember per-index "canonical harness" on
-    // the index metadata. For now: latest wins.)
-    let harness_name = history.last().unwrap().eval_set.clone();
-    let mut harness_set = harness::EvalSet::load(&deps.store, &harness_name).await
+    let mut harness_set = deps.harness_store.load_for_index(&event.index_name, &harness_name).await
         .map_err(|e| format!("load harness '{harness_name}': {e}"))?;
 
     let embeddings = deps.embedding_cache.get_or_load(&event.index_name).await
@@ -393,7 +395,7 @@ async fn run_one_cycle(
     if !harness_set.ground_truth_built {
         harness::compute_ground_truth(&mut harness_set, &embeddings, &deps.ai_client).await
             .map_err(|e| format!("ground truth: {e}"))?;
-        harness_set.save(&deps.store).await.ok();
+        deps.harness_store.save(&harness_set).await.ok();
     }
 
     // Build + bench.
@@ -428,10 +430,13 @@ async fn run_one_cycle(
     let dims = embeddings.first().map(|e| e.vector.len()).unwrap_or(0);
     let memory_bytes = (embeddings.len() * dims * std::mem::size_of::<f32>() + embeddings.len() * 128) as u64;
 
-    let note = match &event.reason {
-        TriggerReason::DatasetAppended { dataset } => format!("agent: dataset_appended({dataset})"),
-        TriggerReason::Manual => "agent: manual".to_string(),
-        TriggerReason::Periodic => "agent: periodic".to_string(),
+    let note = {
+        let base = match &event.reason {
+            TriggerReason::DatasetAppended { dataset } => format!("agent: dataset_appended({dataset})"),
+            TriggerReason::Manual => "agent: manual".to_string(),
+            TriggerReason::Periodic => "agent: periodic".to_string(),
+        };
+        if is_bootstrap { format!("{base} bootstrap") } else { base }
     };
 
     let trial = Trial {
@@ -597,19 +602,92 @@ pub fn propose_next_config(history: &[Trial], champion: Option<&Trial>) -> Optio
 
 // -------- Helpers --------
 
-/// Find an index to poke on a periodic wake. Strategy: the one with the
-/// most recent trial. If nothing's been trialed yet, return None.
+/// Minimum vectors for an index to be worth auto-tuning. Below this,
+/// HNSW's win over brute-force is too small to matter and trial budget
+/// is better spent on bigger indexes.
+const AUTOTUNE_MIN_VECTORS: usize = 1_000;
+
+/// Find an index to poke on a periodic wake. Strategy: least-recently-tuned
+/// wins — pick the index (from `IndexRegistry`, not just promoted ones)
+/// whose most-recent trial is oldest, so trial budget spreads across the
+/// whole portfolio. Never-trialed indexes sort first (None < Some), which
+/// is exactly what we want: bootstrap them on their first visit.
+///
+/// Why not "most recently promoted" (the original strategy): a converged
+/// index like `threat_intel_v1` sits at recall=1.0 on a minimal graph and
+/// can't be improved — but it was always the freshest promotion, so the
+/// agent burned every trial on it while 40+ other indexes got zero
+/// attention. Rotating by last-trial-time fixes that without any explicit
+/// convergence detection.
+///
+/// Why IndexRegistry instead of PromotionRegistry: on a fresh system only
+/// a handful of indexes have ever been promoted, so promotion-based
+/// picking starves new ones. IndexRegistry lists every live index the
+/// platform knows about; filtering by `AUTOTUNE_MIN_VECTORS` keeps the
+/// proposer off indexes where the graph config doesn't matter.
+///
+/// Cost: N extra journal reads per periodic tick, where N = eligible
+/// indexes. Fine at dozens; if the portfolio grows into thousands, cache
+/// `last_trial_at` on the IndexMeta.
 async fn pick_periodic_target(deps: &AgentDeps) -> Option<String> {
-    // `/agent` runs against any index that has a trial journal. We don't
-    // have a "list all journals" helper, so we derive candidates from the
-    // promotion registry (indexes with a human ever promoting are live).
-    let promos = deps.promotion_registry.list_all().await.ok()?;
-    // Prefer the one most recently promoted — it's the one a human cares
-    // about right now.
-    promos.into_iter()
-        .filter_map(|f| f.current.map(|c| (f.index_name, c.promoted_at)))
-        .max_by_key(|(_, at)| *at)
-        .map(|(name, _)| name)
+    let candidates: Vec<String> = deps
+        .index_registry
+        .list(None, None)
+        .await
+        .into_iter()
+        .filter(|m| m.chunk_count >= AUTOTUNE_MIN_VECTORS)
+        .map(|m| m.index_name)
+        .collect();
+    if candidates.is_empty() { return None; }
+
+    let mut scored: Vec<(String, Option<DateTime<Utc>>)> = Vec::with_capacity(candidates.len());
+    for name in candidates {
+        let last_trial_at = deps
+            .trial_journal
+            .list(&name)
+            .await
+            .ok()
+            .and_then(|trials| trials.into_iter().map(|t| t.created_at).max());
+        scored.push((name, last_trial_at));
+    }
+
+    scored.into_iter().min_by_key(|(_, last)| *last).map(|(n, _)| n)
+}
+
+/// On the first visit to an index, load or synthesize an eval harness so
+/// bootstrap trials have something to measure recall against. Returns the
+/// harness name. Synthetic harnesses sample 20 chunks and use their first
+/// 200 chars as self-queries — the ground truth is computed by brute-force
+/// cosine, so recall numbers are real (a config that finds the chunk in
+/// top-k counts as a hit).
+async fn ensure_auto_harness(index_name: &str, deps: &AgentDeps) -> Result<String, String> {
+    let name = format!("{index_name}_auto");
+
+    if deps.harness_store.load_for_index(index_name, &name).await.is_ok() {
+        return Ok(name);
+    }
+
+    let embeddings = deps
+        .embedding_cache
+        .get_or_load(index_name)
+        .await
+        .map_err(|e| format!("load embeddings for auto-harness: {e}"))?;
+    if embeddings.is_empty() {
+        return Err(format!("index '{index_name}' has no embeddings — cannot bootstrap"));
+    }
+
+    let mut eval = harness::synthetic_from_chunks(&name, index_name, &embeddings, 20, 10);
+    harness::compute_ground_truth(&mut eval, &embeddings, &deps.ai_client)
+        .await
+        .map_err(|e| format!("auto-harness ground truth: {e}"))?;
+    deps.harness_store.save(&eval).await
+        .map_err(|e| format!("save auto-harness: {e}"))?;
+
+    tracing::info!(
+        "agent: bootstrapped harness '{}' for index '{}' (20 self-queries, k=10)",
+        name, index_name
+    );
+    Ok(name)
 }
 
 async fn over_rate_limit(inner: &Arc<AgentInner>, cap: u32) -> bool {

crates/vectord/src/autotune.rs

@@ -28,7 +28,7 @@ use aibridge::client::AiClient;
 use catalogd::registry::Registry as CatalogRegistry;
 
 use crate::embedding_cache::EmbeddingCache;
-use crate::harness;
+use crate::harness::{self, HarnessStore};
 use crate::hnsw::HnswStore;
 use crate::index_registry::IndexRegistry;
 use crate::jobs::JobTracker;
@@ -128,7 +128,7 @@ fn pick_winner(trials: &[Trial], min_recall: f32) -> Option<&Trial> {
 #[allow(clippy::too_many_arguments)]
 pub async fn run_autotune(
     req: AutotuneRequest,
-    store: &Arc<dyn ObjectStore>,
+    _store: &Arc<dyn ObjectStore>,
     catalog: &CatalogRegistry,
     ai_client: &AiClient,
     embedding_cache: &EmbeddingCache,
@@ -136,6 +136,7 @@ pub async fn run_autotune(
     index_registry: &IndexRegistry,
     trial_journal: &TrialJournal,
     promotion_registry: &PromotionRegistry,
+    harness_store: &HarnessStore,
     _job_tracker: &JobTracker,
 ) -> Result<AutotuneResult, String> {
     let t0 = std::time::Instant::now();
@@ -146,8 +147,11 @@ pub async fn run_autotune(
     }
     let _ = catalog; // reserved for future audit emission
 
-    // Load the harness once, compute ground truth once.
-    let mut harness_set = harness::EvalSet::load(store, &req.harness)
+    // Load the harness once, compute ground truth once. Harness resolves
+    // to the index's bucket via HarnessStore, with primary as fallback for
+    // pre-federation evals.
+    let mut harness_set = harness_store
+        .load_for_index(&req.index_name, &req.harness)
         .await
         .map_err(|e| format!("load harness: {e}"))?;
 
@@ -160,7 +164,7 @@ pub async fn run_autotune(
         harness::compute_ground_truth(&mut harness_set, &embeddings, ai_client)
             .await
             .map_err(|e| format!("ground truth: {e}"))?;
-        harness_set.save(store).await.ok();
+        harness_store.save(&harness_set).await.ok();
     }
 
     let (grid, rejected) = sanitize_grid(req.grid.clone().unwrap_or_else(default_grid));

crates/vectord/src/harness.rs

@@ -10,11 +10,14 @@
 use chrono::{DateTime, Utc};
 use object_store::ObjectStore;
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
 use std::sync::Arc;
 
 use aibridge::client::{AiClient, EmbedRequest};
 use storaged::ops;
+use storaged::registry::BucketRegistry;
 
+use crate::index_registry::IndexRegistry;
 use crate::store::StoredEmbedding;
 
 /// A single eval query with optional pre-computed ground truth.
@@ -85,6 +88,108 @@ impl EvalSet {
     }
 }
 
+/// Federation-aware wrapper around EvalSet persistence. Mirrors the
+/// `TrialJournal` / `PromotionRegistry` pattern: harness files colocate
+/// with their index's bucket (looked up via `IndexMeta.bucket`), falling
+/// back to `primary` for indexes the registry has never seen. Legacy
+/// harnesses predating federation remain discoverable — lookups transparently
+/// try the resolved bucket first, then `primary` as a fallback. Cross-bucket
+/// listing dedupes across every registered bucket so `GET /hnsw/evals`
+/// returns a complete picture.
+#[derive(Clone)]
+pub struct HarnessStore {
+    buckets: Arc<BucketRegistry>,
+    index_registry: IndexRegistry,
+}
+
+impl HarnessStore {
+    pub fn new(buckets: Arc<BucketRegistry>, index_registry: IndexRegistry) -> Self {
+        Self { buckets, index_registry }
+    }
+
+    /// Resolve which bucket holds this index's eval artifacts. Indexes the
+    /// registry has never heard of fall through to `primary`.
+    async fn bucket_for_index(&self, index_name: &str) -> String {
+        self.index_registry
+            .get(index_name)
+            .await
+            .map(|m| m.bucket)
+            .unwrap_or_else(|| "primary".to_string())
+    }
+
+    /// Save to the bucket that owns `eval.index_name`. Writes under the
+    /// standard `_hnsw_evals/{name}.json` prefix of the resolved bucket.
+    pub async fn save(&self, eval: &EvalSet) -> Result<(), String> {
+        let bucket = self.bucket_for_index(&eval.index_name).await;
+        let store = self.buckets.get(&bucket)?;
+        eval.save(&store).await
+    }
+
+    /// Load a harness by name, given the index it belongs to. Tries the
+    /// index's bucket first; if the file is absent AND the resolved bucket
+    /// isn't `primary`, falls through to `primary` so pre-federation evals
+    /// remain reachable without migration.
+    pub async fn load_for_index(
+        &self,
+        index_name: &str,
+        harness_name: &str,
+    ) -> Result<EvalSet, String> {
+        let bucket = self.bucket_for_index(index_name).await;
+        let primary_store = self.buckets.get("primary")?;
+        let store = self.buckets.get(&bucket)?;
+
+        match EvalSet::load(&store, harness_name).await {
+            Ok(e) => Ok(e),
+            Err(e) if bucket != "primary" => EvalSet::load(&primary_store, harness_name)
+                .await
+                .map_err(|primary_err| format!("{bucket}: {e}; primary fallback: {primary_err}")),
+            Err(e) => Err(e),
+        }
+    }
+
+    /// Find a harness by name without knowing which index it belongs to —
+    /// used by `GET /hnsw/evals/{name}`. Scans every registered bucket;
+    /// first hit wins. Primary is searched first so pre-federation evals
+    /// with the same name as a federated one resolve deterministically.
+    pub async fn get_any(&self, harness_name: &str) -> Result<EvalSet, String> {
+        let bucket_infos = self.buckets.list().await;
+        let mut ordered: Vec<String> = bucket_infos.iter().map(|b| b.name.clone()).collect();
+        ordered.sort_by_key(|n| if n == "primary" { 0 } else { 1 });
+
+        let mut last_err = None;
+        for b in ordered {
+            let store = match self.buckets.get(&b) {
+                Ok(s) => s,
+                Err(e) => { last_err = Some(e); continue; }
+            };
+            match EvalSet::load(&store, harness_name).await {
+                Ok(e) => return Ok(e),
+                Err(e) => { last_err = Some(e); }
+            }
+        }
+        Err(last_err.unwrap_or_else(|| format!("no buckets registered for eval '{harness_name}'")))
+    }
+
+    /// Union of every harness name across every registered bucket.
+    /// Duplicates (same name in multiple buckets — pathological but
+    /// possible after a manual migration) are collapsed.
+    pub async fn list_all(&self) -> Vec<String> {
+        let mut all: HashSet<String> = HashSet::new();
+        for b in self.buckets.list().await {
+            let store = match self.buckets.get(&b.name) {
+                Ok(s) => s,
+                Err(_) => continue,
+            };
+            if let Ok(names) = EvalSet::list(&store).await {
+                all.extend(names);
+            }
+        }
+        let mut out: Vec<String> = all.into_iter().collect();
+        out.sort();
+        out
+    }
+}
+
 /// Cosine similarity for two same-length f32 slices.
 fn cosine(a: &[f32], b: &[f32]) -> f32 {
     let mut dot = 0.0f32;
@@ -127,13 +232,15 @@ pub fn recall_at_k(predicted: &[String], ground_truth: &[String], k: usize) -> f
     if ground_truth.is_empty() || k == 0 {
         return 0.0;
     }
+    // Set-intersection recall@k. Previous implementation counted duplicates
+    // in `predicted` (a corpus with repeated chunks — e.g. cached LLM
+    // responses — returns the same doc_id multiple times via HNSW), which
+    // inflated recall above 1.0 and poisoned promotion decisions.
     let gt_set: std::collections::HashSet<&String> =
         ground_truth.iter().take(k).collect();
-    let hits = predicted
-        .iter()
-        .take(k)
-        .filter(|d| gt_set.contains(d))
-        .count();
+    let pred_set: std::collections::HashSet<&String> =
+        predicted.iter().take(k).collect();
+    let hits = pred_set.intersection(&gt_set).count();
     hits as f32 / gt_set.len() as f32
 }
 
@@ -214,3 +321,67 @@ pub fn synthetic_from_chunks(
         ground_truth_built: false,
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn s(v: &[&str]) -> Vec<String> { v.iter().map(|x| x.to_string()).collect() }
+
+    #[test]
+    fn recall_empty_ground_truth_is_zero() {
+        assert_eq!(recall_at_k(&s(&["a", "b"]), &[], 10), 0.0);
+    }
+
+    #[test]
+    fn recall_k_zero_is_zero() {
+        assert_eq!(recall_at_k(&s(&["a"]), &s(&["a"]), 0), 0.0);
+    }
+
+    #[test]
+    fn recall_perfect_match_equals_one() {
+        let pred = s(&["a", "b", "c"]);
+        let gt = s(&["a", "b", "c"]);
+        assert!((recall_at_k(&pred, &gt, 3) - 1.0).abs() < 1e-6);
+    }
+
+    #[test]
+    fn recall_half_match() {
+        let pred = s(&["a", "b", "x", "y"]);
+        let gt = s(&["a", "b", "c", "d"]);
+        assert!((recall_at_k(&pred, &gt, 4) - 0.5).abs() < 1e-6);
+    }
+
+    #[test]
+    fn recall_duplicates_in_predicted_do_not_inflate() {
+        // Regression guard: the previous implementation counted each
+        // duplicate in `predicted` separately, which could push recall
+        // above 1.0 on corpora with repeated chunks (cached responses etc).
+        // Set-intersection semantics keep it bounded in [0, 1].
+        let pred = s(&["a", "a", "a", "a"]);
+        let gt = s(&["a", "b", "c", "d"]);
+        let r = recall_at_k(&pred, &gt, 4);
+        assert!(r <= 1.0, "recall {r} must not exceed 1.0");
+        // One unique match out of four in gt = 0.25.
+        assert!((r - 0.25).abs() < 1e-6);
+    }
+
+    #[test]
+    fn recall_duplicates_in_ground_truth_handled() {
+        // gt with dupes reduces effective |gt|; matching one hits all.
+        let pred = s(&["x"]);
+        let gt = s(&["x", "x", "x"]);
+        let r = recall_at_k(&pred, &gt, 3);
+        assert!(r <= 1.0);
+        assert!((r - 1.0).abs() < 1e-6); // |pred ∩ gt| / |gt_set| = 1/1
+    }
+
+    #[test]
+    fn recall_respects_k_bound() {
+        // k=2 means only the first 2 of pred and gt count toward the set.
+        let pred = s(&["a", "b", "c", "d"]);
+        let gt = s(&["a", "b", "c", "d"]);
+        let r = recall_at_k(&pred, &gt, 2);
+        assert!((r - 1.0).abs() < 1e-6);
+    }
+}

crates/vectord/src/rag.rs

@@ -10,6 +10,7 @@ use object_store::ObjectStore;
 use std::sync::Arc;
 
 use aibridge::client::{AiClient, EmbedRequest, GenerateRequest};
+use aibridge::continuation::{generate_continuable, ContinuableOpts, ResponseShape};
 use crate::search::{self, SearchResult};
 use crate::store;
 
@@ -42,7 +43,10 @@ async fn rerank(
         system: None,
         temperature: Some(0.0),
         max_tokens: Some(50),
-        think: None,
+        // Reranker returns a comma-separated int list — pure structured
+        // output, zero benefit from hidden reasoning. Opt out to avoid
+        // the empty-response failure mode Phase 21 catalogued.
+        think: Some(false),
     }).await;
 
     match resp {
@@ -151,19 +155,29 @@ pub async fn query(
         Answer:"
     );
 
-    let gen_resp = ai_client.generate(GenerateRequest {
-        prompt,
-        model: None,
-        system: None,
-        temperature: Some(0.2),
-        max_tokens: Some(512),
-        think: None,
-    }).await?;
+    // Route the answer call through Phase 21's generate_continuable so
+    // a thinking-model empty-response or a mid-JSON truncation self-
+    // repairs instead of silently returning half an answer. Shape is
+    // Text (the answer is prose, not JSON), think is Some(false) to
+    // opt out of hidden reasoning on the hot path. This is the first
+    // production caller of the Phase 21 primitives — see audit finding
+    // "Phase 21 Rust primitives are wired but not CALLED by any
+    // production surface" from 2026-04-21.
+    let mut cont_opts = ContinuableOpts::new("qwen2.5:latest");
+    cont_opts.max_tokens = Some(512);
+    cont_opts.temperature = Some(0.2);
+    cont_opts.shape = ResponseShape::Text;
+    cont_opts.think = Some(false);
+    let outcome = generate_continuable(ai_client, &prompt, &cont_opts).await?;
 
     Ok(RagResponse {
-        answer: gen_resp.text.trim().to_string(),
-        model: gen_resp.model,
+        answer: outcome.text.trim().to_string(),
+        // generate_continuable doesn't surface the model name (sidecar
+        // echoes whatever Ollama loaded). Use the configured tier model
+        // for now; if RAG needs to report the actual resolved model,
+        // the runner can add a post-call ps probe later.
+        model: "qwen2.5:latest".to_string(),
         sources: results,
-        tokens_generated: gen_resp.tokens_generated,
+        tokens_generated: None,
     })
 }

crates/vectord/src/service.rs

@@ -13,6 +13,7 @@ use aibridge::client::{AiClient, EmbedRequest, GenerateRequest};
 use catalogd::registry::Registry as CatalogRegistry;
 use storaged::registry::BucketRegistry;
 use crate::{agent, autotune, chunker, embedding_cache, harness, hnsw, index_registry, jobs, lance_backend, playbook_memory, promotion, rag, refresh, search, store, supervisor, trial};
+use tokio::sync::Semaphore;
 
 #[derive(Clone)]
 pub struct VectorState {
@@ -53,6 +54,9 @@ pub struct VectorState {
     /// and, when `use_playbook_memory` is set on /vectors/hybrid, boosts
     /// workers that were actually filled in semantically-similar past ops.
     pub playbook_memory: playbook_memory::PlaybookMemory,
+    /// Serializes embed calls from seed_playbook_memory to avoid
+    /// concurrent socket collisions with the Python sidecar.
+    pub embed_semaphore: Arc<Semaphore>,
 }
 
 /// What the active-profile singleton records. Narrow — we don't need the
@@ -893,7 +897,9 @@ async fn hybrid_search(
             system: None,
             temperature: Some(0.2),
             max_tokens: Some(512),
-            think: None,
+            // Hybrid's answer step — prose output over retrieved records,
+            // no reasoning needed on the hot path.
+            think: Some(false),
         }).await;
 
         gen_resp.ok().map(|r| r.text.trim().to_string())
@@ -1583,7 +1589,7 @@ async fn activate_profile(
         });
     }
 
-    Ok(Json(ActivateReport {
+    let report = ActivateReport {
         profile_id: profile.id,
         ollama_name: profile.ollama_name,
         indexes_warmed: warmed,
@@ -1592,7 +1598,9 @@ async fn activate_profile(
         duration_secs: t0.elapsed().as_secs_f32(),
         model_preloaded,
         previous_profile: previous_slot.map(|s| s.profile_id),
-    }))
+    };
+
+    Ok(Json(report))
 }
 
 /// Unload this profile's model and clear the active slot. No-op if the
@@ -2217,6 +2225,10 @@ async fn seed_playbook_memory(
     State(state): State<VectorState>,
     Json(req): Json<SeedPlaybookRequest>,
 ) -> impl IntoResponse {
+    // Serialize embed calls to avoid concurrent socket collisions.
+    let _permit = state.embed_semaphore.acquire().await.map_err(|e|
+        (StatusCode::INTERNAL_SERVER_ERROR, format!("semaphore error: {e}")))?;
+
     // Embed the entry through the same text shape `rebuild` uses so
     // similarity math is comparable across seed + real entries.
     let tmp_entry = playbook_memory::PlaybookEntry {
@@ -2549,17 +2561,47 @@ async fn revise_playbook_memory(
             "operation must match 'fill: Role xN in City, ST' shape".into()));
     }
 
-    let ts = chrono::Utc::now().to_rfc3339();
+    // Phase 27 — deterministic pid derived ONLY from content-shaped
+    // inputs (parent_id + operation + approach + context + sorted
+    // endorsed_names). Excluding wall-clock ts means two revise calls
+    // with identical payloads produce the same pid, which is the
+    // contract the docstring promises. On retry the caller gets back
+    // the same id and the pre-flight idempotency check below short-
+    // circuits to the existing entry instead of re-appending.
+    let mut names_sorted = req.endorsed_names.clone();
+    names_sorted.sort();
     use sha2::{Digest, Sha256};
     let mut h = Sha256::new();
-    h.update(ts.as_bytes());
-    h.update(b"|");
-    h.update(req.parent_id.as_bytes());
-    h.update(b"|");
-    h.update(req.operation.as_bytes());
+    h.update(req.parent_id.as_bytes()); h.update(b"|");
+    h.update(req.operation.as_bytes()); h.update(b"|");
+    h.update(req.approach.as_bytes());  h.update(b"|");
+    h.update(req.context.as_bytes());   h.update(b"|");
+    h.update(names_sorted.join(",").as_bytes());
     let bytes = h.finalize();
     let pid = format!("pb-rev-{}", bytes.iter().take(8).map(|b| format!("{b:02x}")).collect::<String>());
 
+    // Idempotency short-circuit — if this exact pid already exists in
+    // memory (from a prior successful revise with the same content),
+    // return it directly rather than re-appending or 400ing on the
+    // superseded-parent rejection. Walks the parent's chain via
+    // history() because the parent may itself have been superseded by
+    // our prior successful call.
+    let chain = state.playbook_memory.history(&req.parent_id).await;
+    if let Some(existing) = chain.iter().find(|e| e.playbook_id == pid) {
+        return Ok(Json(serde_json::json!({
+            "outcome": {
+                "parent_id": req.parent_id,
+                "parent_version": existing.version.saturating_sub(1),
+                "new_playbook_id": existing.playbook_id,
+                "new_version": existing.version,
+                "superseded_at": existing.superseded_at.clone().unwrap_or_default(),
+                "idempotent_return": true,
+            },
+            "entries_after": state.playbook_memory.entry_count().await,
+        })));
+    }
+
+    let ts = chrono::Utc::now().to_rfc3339();
     let new_entry = playbook_memory::PlaybookEntry {
         playbook_id: pid.clone(),
         operation: req.operation,