ADR-020: Universal ID mapping — fix the flat embedding identity problem

THE REAL PROBLEM: Every new data source produces different doc_id
prefixes in vector indexes (W-, W500K-, W5K-, CAND-). Hybrid search
had to hardcode strip_prefix for each one. New datasets broke hybrid
until someone added another prefix. This violates "any data source
without pre-defined schemas."

THE FIX: IndexMeta.id_prefix — the catalog records what prefix each
index uses. Hybrid search reads it and strips automatically. Legacy
indexes fall back to heuristic stripping. New indexes can set
id_prefix=None to use raw IDs (no prefix, no stripping needed).

This means: ingest a new dataset, embed it, hybrid search works
immediately without code changes. The system is truly source-agnostic.

Also: full ADR document at docs/ADR-020-universal-id-mapping.md
with the three options considered and rationale for the chosen approach.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/vectord/src/index_registry.rs

@@ -40,6 +40,12 @@ pub struct IndexMeta {
     /// `Lance` means it points at a Lance dataset directory.
     #[serde(default)]
     pub vector_backend: shared::types::VectorBackend,
+    /// ADR-020: prefix prepended to doc_ids during embedding. If set,
+    /// hybrid search strips this prefix to match against SQL primary keys.
+    /// None = doc_ids ARE the raw primary keys (no stripping needed).
+    /// Existing indexes: "W-", "CAND-", "W500K-", etc.
+    #[serde(default)]
+    pub id_prefix: Option<String>,
 }
 
 fn default_bucket() -> String { "primary".to_string() }

crates/vectord/src/refresh.rs

@@ -307,6 +307,7 @@ async fn try_update_index_meta(
         chunks_per_sec: 0.0,
         bucket: "primary".to_string(),
         vector_backend: shared::types::VectorBackend::Parquet,
+        id_prefix: None,
     };
     index_registry.register(meta).await
 }

crates/vectord/src/service.rs

@@ -210,6 +210,7 @@ async fn create_index(
                     chunks_per_sec: rate,
                     bucket: bucket.clone(),
                     vector_backend: shared::types::VectorBackend::Parquet,
+        id_prefix: None,
                 };
                 let _ = registry.register(meta).await;
 
@@ -548,15 +549,28 @@ async fn hybrid_search(
     };
 
     // Step 3: Filter vector results to only SQL-verified IDs.
+    // ADR-020: read the index's id_prefix from the catalog instead of
+    // hardcoding prefix stripping. Falls back to heuristic for legacy indexes.
+    let id_prefix: Option<String> = state.index_registry
+        .get(&req.index_name).await
+        .and_then(|m| m.id_prefix.clone());
+
     let sql_count = valid_ids.as_ref().map(|s| s.len()).unwrap_or(0);
     let filtered: Vec<search::SearchResult> = if let Some(ref ids) = valid_ids {
         all_results.into_iter()
             .filter(|r| {
-                // doc_id format is "W-{worker_id}" — extract the number
-                let num = r.doc_id.strip_prefix("W-")
-                    .or_else(|| r.doc_id.strip_prefix("CAND-"))
-                    .unwrap_or(&r.doc_id);
-                ids.contains(num)
+                let raw_id = if let Some(ref prefix) = id_prefix {
+                    r.doc_id.strip_prefix(prefix.as_str()).unwrap_or(&r.doc_id)
+                } else {
+                    // Legacy: heuristic strip for pre-ADR-020 indexes
+                    r.doc_id.strip_prefix("W500K-")
+                        .or_else(|| r.doc_id.strip_prefix("W500-"))
+                        .or_else(|| r.doc_id.strip_prefix("W5K-"))
+                        .or_else(|| r.doc_id.strip_prefix("W-"))
+                        .or_else(|| r.doc_id.strip_prefix("CAND-"))
+                        .unwrap_or(&r.doc_id)
+                };
+                ids.contains(raw_id)
             })
             .take(req.top_k)
             .collect()