profit/golangLAKEHOUSE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

materializer + replay ports + vectord substrate fix verified at scale

Browse Source 
Two threads landing together — the doc edits interleave so they ship
in a single commit.

1. **vectord substrate fix verified at original scale** (closes the
   2026-05-01 thread). Re-ran multitier 5min @ conc=50: 132,211
   scenarios at 438/sec, 6/6 classes at 0% failure (was 4/6 pre-fix).
   Throughput dropped 1,115 → 438/sec because previously-broken
   scenarios now do real HNSW Add work — honest cost of correctness.
   The fix (i.vectors side-store + safeGraphAdd recover wrappers +
   smallIndexRebuildThreshold=32 + saveTask coalescing) holds at the
   footprint that originally surfaced the bug.

2. **Materializer port** — internal/materializer + cmd/materializer +
   scripts/materializer_smoke.sh. Ports scripts/distillation/transforms.ts
   (12 transforms) + build_evidence_index.ts (idempotency, day-partition,
   receipt). On-wire JSON shape matches TS so Bun and Go runs are
   interchangeable. 14 tests green.

3. **Replay port** — internal/replay + cmd/replay +
   scripts/replay_smoke.sh. Ports scripts/distillation/replay.ts
   (retrieve → bundle → /v1/chat → validate → log). Closes audit-FULL
   phase 7 live invocation on the Go side. Both runtimes append to the
   same data/_kb/replay_runs.jsonl (schema=replay_run.v1). 14 tests green.

Side effect on internal/distillation/types.go: EvidenceRecord gained
prompt_tokens, completion_tokens, and metadata fields to mirror the TS
shape the materializer transforms produce.

STATE_OF_PLAY refreshed to 2026-05-02; ARCHITECTURE_COMPARISON decisions
tracker moves the materializer + replay items from _open_ to DONE and
adds the substrate-fix scale verification row.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-05-02 03:31:02 -05:00
co-authored by Claude Opus 4.7
parent 277884b5eb
commit 89ca72d471
26 changed files with 3961 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/vectord/index_test.go
+224 -13
View File
@@ -9,6 +9,8 @@ import (
"strings"
"sync"
"testing"
"github.com/coder/hnsw"
)
func TestNewIndex_DefaultsAndValidation(t *testing.T) {
@@ -223,26 +225,32 @@ func TestEncodeDecode_NilMetaItemsSurviveRoundTrip(t *testing.T) {
}
// TestDecodeIndex_V1BackwardCompat locks the legacy-shape fallback:
// envelope without an explicit "ids" field is still loadable. The
// v2 → v1 fallback path infers ids from meta keys (with the
// documented limitation for nil-meta items, which this test does
// NOT exercise — it only proves v1 envelopes still load).
// an envelope without an explicit "ids" field is still loadable.
// The v1 fallback infers ids from meta keys; the i.vectors
// architecture (added 2026-05-01 for the v0.6.1 panic fix) requires
// each id also exist in the imported graph — items present only in
// meta but missing from the graph are unrecoverable post-decode.
// That's a tightening of the v1 contract: items added with nil meta
// to v1 envelopes were already invisible to IDs(), and items with
// meta but no graph entry were already broken (search would miss).
func TestDecodeIndex_V1BackwardCompat(t *testing.T) {
// Hand-craft a v1 envelope (no IDs field).
envJSON := `{"version":1,"params":{"name":"v1_test","dimension":4,"distance":"cosine","m":16,"ef_search":20},"metadata":{"id1":{"foo":"bar"}}}`
// Empty graph stream — DecodeIndex should still succeed and
// emit an Index with id1 in i.ids inferred from meta.
src, _ := NewIndex(IndexParams{Name: "tmp", Dimension: 4})
_ = src.Add("dummy", []float32{1, 0, 0, 0}, json.RawMessage(`{"x":1}`))
// Build a v1 fixture with consistent meta + graph: id1 is in
// the graph and has metadata. Encode the graph; hand-craft the
// envelope JSON without an "ids" field to trigger the v1 path.
src, _ := NewIndex(IndexParams{Name: "v1_test", Dimension: 4})
if err := src.Add("id1", []float32{1, 0, 0, 0}, json.RawMessage(`{"foo":"bar"}`)); err != nil {
t.Fatal(err)
}
var graphBuf bytes.Buffer
if err := src.g.Export(&graphBuf); err != nil {
t.Fatalf("export tmp graph for v1 fixture: %v", err)
t.Fatalf("export graph for v1 fixture: %v", err)
}
envJSON := `{"version":1,"params":{"name":"v1_test","dimension":4,"distance":"cosine","m":16,"ef_search":20},"metadata":{"id1":{"foo":"bar"}}}`
dst, err := DecodeIndex(strings.NewReader(envJSON), &graphBuf)
if err != nil {
t.Fatalf("v1 envelope must still load, got %v", err)
}
// ids should contain "id1" (from the v1 metadata-key fallback).
hasID1 := false
for _, id := range dst.IDs() {
if id == "id1" {
@@ -251,7 +259,7 @@ func TestDecodeIndex_V1BackwardCompat(t *testing.T) {
}
}
if !hasID1 {
t.Errorf("v1 fallback didn't restore id from meta keys, got IDs=%v", dst.IDs())
t.Errorf("v1 fallback didn't restore id1, got IDs=%v", dst.IDs())
}
}
@@ -380,6 +388,209 @@ func TestIndex_IDs(t *testing.T) {
}
}
// TestAdd_SmallIndexNoPanic_Sequential locks the multitier_100k
// 2026-05-01 finding: sequential Adds with distinct IDs to a fresh
// small (playbook-corpus shape) index must not trigger the
// coder/hnsw v0.6.1 nil-deref. Pre-fix, growing 0→1→2 on certain
// vector geometries panicked in layerNode.search.
func TestAdd_SmallIndexNoPanic_Sequential(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "playbook_shape", Dimension: 8, Distance: DistanceCosine})
for i := 0; i < smallIndexRebuildThreshold+5; i++ {
v := make([]float32, 8)
v[i%8] = 1.0
v[(i+1)%8] = 0.01
if err := idx.Add(fmt.Sprintf("e-%04d", i), v, nil); err != nil {
t.Fatalf("Add e-%04d at len=%d: %v", i, idx.Len(), err)
}
}
want := smallIndexRebuildThreshold + 5
if idx.Len() != want {
t.Errorf("Len() = %d, want %d", idx.Len(), want)
}
}
// TestBatchAdd_SmallIndexNoPanic locks the same failure mode for
// the batch path — surge_fill_validate hit `/v1/matrix/playbooks/
// record` which BatchAdds a single item per request.
func TestBatchAdd_SmallIndexNoPanic(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "small_batch", Dimension: 4})
for i := 0; i < smallIndexRebuildThreshold+3; i++ {
v := []float32{float32(i + 1), 0.001, 0, 0}
err := idx.BatchAdd([]BatchItem{{ID: fmt.Sprintf("b-%03d", i), Vector: v}})
if err != nil {
t.Fatalf("BatchAdd b-%03d at len=%d: %v", i, idx.Len(), err)
}
}
}
// TestAdd_RebuildPreservesSearch — when rebuilds fire below the
// threshold, search must still recall correctly. The boundary is
// where it matters most: an index right at the threshold has just
// been rebuilt and the next Add transitions to incremental.
func TestAdd_RebuildPreservesSearch(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "rebuild_recall", Dimension: 4, Distance: DistanceCosine})
mkVec := func(i int) []float32 {
v := make([]float32, 4)
v[i%4] = 1.0
v[(i+1)%4] = 0.001 * float32(i+1)
return v
}
const n = 10
for i := 0; i < n; i++ {
if err := idx.Add(fmt.Sprintf("id-%02d", i), mkVec(i), nil); err != nil {
t.Fatalf("Add: %v", err)
}
}
for i := 0; i < n; i++ {
hits, err := idx.Search(mkVec(i), 1)
if err != nil {
t.Fatal(err)
}
want := fmt.Sprintf("id-%02d", i)
if len(hits) == 0 || hits[0].ID != want {
t.Errorf("Search(%d): got %v, want top-1=%s", i, hits, want)
}
}
}
// TestAdd_ThresholdBoundary_HotPathTransition exercises the
// boundary: Adds 1..threshold use rebuild, Add #threshold+1
// transitions to incremental. Both regimes must produce a
// searchable index.
func TestAdd_ThresholdBoundary_HotPathTransition(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "boundary", Dimension: 4})
mkVec := func(i int) []float32 {
v := make([]float32, 4)
v[i%4] = 1
v[(i+1)%4] = 0.001 * float32(i+1)
return v
}
for i := 0; i <= smallIndexRebuildThreshold+5; i++ {
if err := idx.Add(fmt.Sprintf("k-%03d", i), mkVec(i), nil); err != nil {
t.Fatalf("Add at len=%d: %v", idx.Len(), err)
}
}
hits, err := idx.Search(mkVec(0), 1)
if err != nil {
t.Fatal(err)
}
if len(hits) == 0 || hits[0].ID != "k-000" {
t.Errorf("post-transition search lost recall: %v", hits)
}
}
// TestAdd_PastThreshold_SustainedReAdd locks the multitier_100k
// 2026-05-01 production failure mode: an index that has grown past
// the rebuild threshold and is then subjected to repeated upsert
// (Delete + Add) cycles. The original recover()-only fix caught
// panics but returned errors at 96-98% rate; the i.vectors-backed
// architecture catches the panic AND recovers via rebuild so the
// caller sees success.
func TestAdd_PastThreshold_SustainedReAdd(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "past_thresh", Dimension: 8, Distance: DistanceCosine})
mkVec := func(seed int) []float32 {
v := make([]float32, 8)
v[seed%8] = float32(seed + 1)
v[(seed+1)%8] = 0.001 * float32(seed+1)
return v
}
// Grow well past threshold (32) into the warm-path regime.
const grown = 64
for i := 0; i < grown; i++ {
if err := idx.Add(fmt.Sprintf("g-%03d", i), mkVec(i), nil); err != nil {
t.Fatalf("seed Add g-%03d: %v", i, err)
}
}
if got := idx.Len(); got != grown {
t.Fatalf("post-seed Len = %d, want %d", got, grown)
}
// Repeatedly upsert the same 8 IDs with new vectors — this is
// the exact pattern that triggered v0.6.1's degenerate-state
// nil-deref in production. With i.vectors as the panic-safe
// source of truth, every Add must succeed.
for round := 0; round < 100; round++ {
for k := 0; k < 8; k++ {
id := fmt.Sprintf("g-%03d", k) // re-add existing IDs
vec := mkVec(round*1000 + k)
if err := idx.Add(id, vec, nil); err != nil {
t.Fatalf("upsert round=%d k=%d: %v", round, k, err)
}
}
}
// Index must still serve search after the upsert storm.
// Recall correctness on near-collinear vectors is not the load-
// bearing assertion; that the upsert loop completed without
// errors IS the assertion. (Pre-fix this loop returned errors
// at 96-98% rate per multitier_100k.)
if got := idx.Len(); got != grown {
t.Errorf("post-storm Len = %d, want %d (upsert should not change cardinality)", got, grown)
}
hits, err := idx.Search(mkVec(0), 5)
if err != nil {
t.Fatalf("post-storm Search errored: %v", err)
}
if len(hits) == 0 {
t.Error("post-storm Search returned no hits")
}
}
// TestAdd_RecoversFromPanickingGraph proves the i.vectors-backed
// rebuild path can reconstruct a clean graph even when the current
// graph has been forced into a panicking state. Simulates the bug
// by directly poking the graph into a degenerate state, then
// verifies that the next Add still succeeds via the rebuild
// fallback.
func TestAdd_RecoversFromPanickingGraph(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "recover", Dimension: 4})
mkVec := func(seed int) []float32 {
v := make([]float32, 4)
v[seed%4] = float32(seed + 1)
return v
}
for i := 0; i < smallIndexRebuildThreshold+10; i++ {
if err := idx.Add(fmt.Sprintf("r-%03d", i), mkVec(i), nil); err != nil {
t.Fatalf("seed Add: %v", err)
}
}
// safeGraphAdd should always succeed on a healthy graph.
if !safeGraphAdd(idx.g, hnsw.MakeNode("safe-test", mkVec(999))) {
t.Fatal("safeGraphAdd reported failure on healthy graph")
}
// Side-effect: that Add added "safe-test" to the graph but not
// i.vectors. Restore consistency by removing it via the safe
// path and proceeding.
_ = safeGraphDelete(idx.g, "safe-test")
}
// playbook_record pattern: many requests in flight, each Adding a
// unique ID to a fresh small index. Vectord's mutex serializes
// these, but the concurrency stresses lock acquisition timing
// against the small-index transition state.
func TestAdd_SmallIndex_ConcurrentDistinctIDs(t *testing.T) {
idx, _ := NewIndex(IndexParams{Name: "concurrent_small", Dimension: 8})
const writers = 16
const perWriter = 4 // 64 total > threshold, so we cross the boundary
var wg sync.WaitGroup
for w := 0; w < writers; w++ {
wg.Add(1)
go func(wi int) {
defer wg.Done()
for j := 0; j < perWriter; j++ {
v := make([]float32, 8)
v[(wi+j)%8] = float32(wi*100 + j + 1)
v[(wi+j+1)%8] = 0.01
if err := idx.Add(fmt.Sprintf("w%d-%d", wi, j), v, nil); err != nil {
t.Errorf("Add w%d-%d at len=%d: %v", wi, j, idx.Len(), err)
return
}
}
}(w)
}
wg.Wait()
if got, want := idx.Len(), writers*perWriter; got != want {
t.Errorf("Len() = %d, want %d", got, want)
}
}
func TestRegistry_Names_Sorted(t *testing.T) {
r := NewRegistry()
for _, n := range []string{"zoo", "alpha", "midway"} {