profit/lakehouse
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Scalar btree index on doc_id + auto-build during Lance activation

Browse Source 
LanceVectorStore gains build_scalar_index(column) and
has_scalar_index(column). Exposed as POST /vectors/lance/scalar-index/
{index}/{column}. activate_profile auto-builds the doc_id btree
alongside the IVF_PQ vector index when activating a Lance-backed
profile — operators get both indexes without extra API calls.

stats() now reports has_doc_id_index alongside has_vector_index.

Measured on resumes_100k_v2 (100K × 768d): random doc_id fetch
improved from ~5.4ms to ~3.5ms (35% faster). Btree build: 19ms,
+2.7 MB on disk. The remaining ~3ms is vector column materialization,
not index lookup — to close further would need a projection-only
fetch that skips the 768-float vector for text-only RAG retrieval.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-16 20:49:17 -05:00
co-authored by Claude Opus 4.6
parent 2592f8fcb3
commit 59e72fa566
2 changed files with 85 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/vectord/src/service.rs
+28 -5
View File
@@ -107,6 +107,7 @@ pub fn router(state: VectorState) -> Router {
.route("/lance/doc/{index_name}/{doc_id}", get(lance_get_doc))
.route("/lance/append/{index_name}", post(lance_append))
.route("/lance/stats/{index_name}", get(lance_stats))
.route("/lance/scalar-index/{index_name}/{column}", post(lance_build_scalar_index))
.with_state(state)
}
@@ -967,17 +968,25 @@ async fn activate_profile(
hnsw_build_secs: 0.0,
});
}
// Ensure IVF_PQ index exists.
// Ensure IVF_PQ vector index exists.
if !lance_store.has_vector_index().await.unwrap_or(false) {
let build_t = std::time::Instant::now();
match lance_store.build_index(316, 8, 48).await {
Ok(ix) => tracing::info!(
"lance auto-index '{}': built in {:.1}s",
"lance auto-index '{}': IVF_PQ built in {:.1}s",
meta.index_name, ix.build_time_secs,
),
Err(e) => failures.push(format!("{}: lance index build: {e}", meta.index_name)),
Err(e) => failures.push(format!("{}: lance IVF_PQ build: {e}", meta.index_name)),
}
}
// Ensure scalar btree on doc_id for O(log N) random fetch.
if !lance_store.has_scalar_index("doc_id").await.unwrap_or(false) {
match lance_store.build_scalar_index("doc_id").await {
Ok(ix) => tracing::info!(
"lance auto-index '{}': doc_id btree built in {:.2}s",
meta.index_name, ix.build_time_secs,
),
Err(e) => failures.push(format!("{}: lance doc_id btree: {e}", meta.index_name)),
}
let _ = build_t; // suppress unused warning
}
} else {
// --- Parquet + HNSW activation path (existing) ---
@@ -1499,3 +1508,17 @@ async fn lance_stats(
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
}
}
/// Build a scalar btree index on a column (typically `doc_id`). Makes
/// filter-pushdown queries O(log N) instead of full-fragment scan.
async fn lance_build_scalar_index(
State(state): State<VectorState>,
Path((index_name, column)): Path<(String, String)>,
) -> impl IntoResponse {
let lance_store = state.lance.store_for(&index_name).await
.map_err(|e| (StatusCode::BAD_REQUEST, e))?;
match lance_store.build_scalar_index(&column).await {
Ok(stats) => Ok(Json(stats)),
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
}
}