materializer + replay ports + vectord substrate fix verified at scale

Two threads landing together — the doc edits interleave so they ship
in a single commit.

1. **vectord substrate fix verified at original scale** (closes the
   2026-05-01 thread). Re-ran multitier 5min @ conc=50: 132,211
   scenarios at 438/sec, 6/6 classes at 0% failure (was 4/6 pre-fix).
   Throughput dropped 1,115 → 438/sec because previously-broken
   scenarios now do real HNSW Add work — honest cost of correctness.
   The fix (i.vectors side-store + safeGraphAdd recover wrappers +
   smallIndexRebuildThreshold=32 + saveTask coalescing) holds at the
   footprint that originally surfaced the bug.

2. **Materializer port** — internal/materializer + cmd/materializer +
   scripts/materializer_smoke.sh. Ports scripts/distillation/transforms.ts
   (12 transforms) + build_evidence_index.ts (idempotency, day-partition,
   receipt). On-wire JSON shape matches TS so Bun and Go runs are
   interchangeable. 14 tests green.

3. **Replay port** — internal/replay + cmd/replay +
   scripts/replay_smoke.sh. Ports scripts/distillation/replay.ts
   (retrieve → bundle → /v1/chat → validate → log). Closes audit-FULL
   phase 7 live invocation on the Go side. Both runtimes append to the
   same data/_kb/replay_runs.jsonl (schema=replay_run.v1). 14 tests green.

Side effect on internal/distillation/types.go: EvidenceRecord gained
prompt_tokens, completion_tokens, and metadata fields to mirror the TS
shape the materializer transforms produce.

STATE_OF_PLAY refreshed to 2026-05-02; ARCHITECTURE_COMPARISON decisions
tracker moves the materializer + replay items from _open_ to DONE and
adds the substrate-fix scale verification row.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

cmd/materializer/main.go

@@ -0,0 +1,78 @@
+// materializer — Go-side build_evidence_index runner. Reads source
+// JSONL streams in `data/_kb/`, transforms each row to an
+// EvidenceRecord, writes day-partitioned output under `data/evidence/`
+// + an audit-grade receipt under `reports/distillation/<ts>/`.
+//
+// Mirrors the Bun runner at scripts/distillation/build_evidence_index.ts
+// — both runtimes can run against the same root and produce
+// interoperable outputs (per ADR-001 #4: same logic, on-wire
+// JSON shape preserved).
+//
+// Usage:
+//
+//	materializer                                        # full run, write outputs
+//	materializer -dry-run                               # count, no writes
+//	materializer -root /home/profit/lakehouse           # custom repo root
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"git.agentview.dev/profit/golangLAKEHOUSE/internal/materializer"
+)
+
+func main() {
+	root := flag.String("root", defaultRoot(), "lakehouse repo root (defaults to $LH_DISTILL_ROOT or current dir)")
+	dryRun := flag.Bool("dry-run", false, "count rows but do not write outputs")
+	flag.Parse()
+
+	recordedAt := time.Now().UTC().Format(time.RFC3339Nano)
+
+	res, err := materializer.MaterializeAll(materializer.MaterializeOptions{
+		Root:       *root,
+		Transforms: materializer.Transforms,
+		RecordedAt: recordedAt,
+		DryRun:     *dryRun,
+	})
+	if err != nil {
+		log.Fatalf("materializer: %v", err)
+	}
+
+	suffix := ""
+	if *dryRun {
+		suffix = " (DRY RUN)"
+	}
+	fmt.Printf("[evidence_index] %d read · %d written · %d skipped · %d deduped%s\n",
+		res.Totals.RowsRead, res.Totals.RowsWritten, res.Totals.RowsSkipped, res.Totals.RowsDeduped, suffix)
+	for _, s := range res.Sources {
+		if !s.RowsPresent {
+			fmt.Printf("  %s: (missing — skipped)\n", s.SourceFileRelPath)
+			continue
+		}
+		fmt.Printf("  %s: read=%d wrote=%d skip=%d dedup=%d\n",
+			s.SourceFileRelPath, s.RowsRead, s.RowsWritten, s.RowsSkipped, s.RowsDeduped)
+	}
+
+	if !*dryRun {
+		fmt.Printf("[evidence_index] receipt: %s\n", res.ReceiptPath)
+		fmt.Printf("[evidence_index] validation_pass=%v\n", res.Receipt.ValidationPass)
+	}
+
+	if !res.Receipt.ValidationPass {
+		os.Exit(1)
+	}
+}
+
+func defaultRoot() string {
+	if r := os.Getenv("LH_DISTILL_ROOT"); r != "" {
+		return r
+	}
+	if cwd, err := os.Getwd(); err == nil {
+		return cwd
+	}
+	return "."
+}

cmd/replay/main.go

@@ -0,0 +1,87 @@
+// replay — Go-side distillation replay runner. Closes audit-FULL
+// phase 7 live invocation on the Go side. Mirrors
+// scripts/distillation/replay.ts; both runtimes append to the same
+// `data/_kb/replay_runs.jsonl` shape (schema=replay_run.v1).
+//
+// Usage:
+//
+//	replay -task "rebuild evidence index"
+//	replay -task "..."  -allow-escalation
+//	replay -task "..."  -no-retrieval                # baseline mode
+//	replay -task "..."  -dry-run                     # synthetic, no LLM
+//	replay -task "..."  -root /home/profit/lakehouse # custom repo root
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"git.agentview.dev/profit/golangLAKEHOUSE/internal/replay"
+)
+
+func main() {
+	task := flag.String("task", "", "input task to replay")
+	localOnly := flag.Bool("local-only", false, "never escalate; record validation result only")
+	allowEscalation := flag.Bool("allow-escalation", false, "fall back to the bigger model when local validation fails")
+	noRetrieval := flag.Bool("no-retrieval", false, "baseline mode: skip retrieval bundle (still logs)")
+	dryRun := flag.Bool("dry-run", false, "synthesize a deterministic response — no LLM call")
+	root := flag.String("root", replay.DefaultRoot(), "lakehouse repo root (defaults to $LH_DISTILL_ROOT or cwd)")
+	gateway := flag.String("gateway", "", "override gateway URL (default: $LH_GATEWAY_URL or http://localhost:3110)")
+	localModel := flag.String("local-model", "", "override local model name")
+	escalationModel := flag.String("escalation-model", "", "override escalation model name")
+	flag.Parse()
+
+	if *task == "" {
+		fmt.Fprintln(os.Stderr, `usage: replay -task "<input>" [-local-only] [-allow-escalation] [-no-retrieval] [-dry-run]`)
+		os.Exit(2)
+	}
+
+	res, err := replay.Replay(context.Background(), replay.ReplayRequest{
+		Task:            *task,
+		LocalOnly:       *localOnly,
+		AllowEscalation: *allowEscalation,
+		NoRetrieval:     *noRetrieval,
+		DryRun:          *dryRun,
+		GatewayURL:      *gateway,
+		LocalModel:      *localModel,
+		EscalationModel: *escalationModel,
+	}, *root)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "replay: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("[replay] run_id=%s\n", res.RecordedRunID)
+	if res.ContextBundle == nil {
+		fmt.Println("[replay] retrieval: DISABLED")
+	} else {
+		fmt.Printf("[replay] retrieval: %d playbooks\n", len(res.ContextBundle.RetrievedPlaybooks))
+	}
+	fmt.Printf("[replay] escalation_path: %s\n", strings.Join(res.EscalationPath, " → "))
+	fmt.Printf("[replay] model_used: %s · %dms\n", res.ModelUsed, res.DurationMs)
+	verdict := "PASS"
+	if !res.ValidationResult.Passed {
+		verdict = "FAIL"
+	}
+	suffix := ""
+	if len(res.ValidationResult.Reasons) > 0 {
+		suffix = " (" + strings.Join(res.ValidationResult.Reasons, "; ") + ")"
+	}
+	fmt.Printf("[replay] validation: %s%s\n", verdict, suffix)
+	fmt.Println()
+	fmt.Println("─── response ───")
+	body := res.ModelResponse
+	if len(body) > 1500 {
+		fmt.Println(body[:1500])
+		fmt.Printf("... [%d more chars]\n", len(body)-1500)
+	} else {
+		fmt.Println(body)
+	}
+
+	if !res.ValidationResult.Passed {
+		os.Exit(1)
+	}
+}

cmd/vectord/main.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/go-chi/chi/v5"
@@ -71,6 +72,73 @@ func main() {
 type handlers struct {
 	reg     *vectord.Registry
 	persist *vectord.Persistor // nil when persistence is disabled
+
+	// saversMu guards lazy initialization of per-index save tasks.
+	// Each task coalesces synchronous Save calls into single-flight
+	// async saves so high-write-rate indexes (playbook_memory under
+	// multitier_100k load) don't pay one MinIO PUT per Add. See the
+	// saveTask docstring for the coalescing semantics.
+	saversMu sync.Mutex
+	savers   map[string]*saveTask
+}
+
+// saveTask coalesces saves for one index into a single-flight async
+// goroutine. While a save is in-flight, additional triggers mark
+// "pending" — the in-flight goroutine reruns the save after it
+// finishes, collapsing N concurrent triggers into at most 2 saves
+// (the current in-flight + one catch-up).
+//
+// Why: pre-2026-05-01 each successful Add called Persistor.Save
+// synchronously inside the request handler. For playbook_memory at
+// 1900-entry / 768-d, Encode + MinIO PUT cost 100-300ms. With 50
+// concurrent writers, end-to-end Add latency hit 2-2.5s purely from
+// save serialization (Save takes the index RLock for Encode, which
+// blocks new Adds taking the Lock).
+//
+// Trade-off: RPO. Add now returns OK before the save completes, so
+// a crash can lose up to ~1 save's worth of data. Acceptable for
+// the playbook-memory shape (learning loop — lost trace re-recorded
+// on next run) and consistent with ADR-005's fail-open posture.
+type saveTask struct {
+	mu       sync.Mutex
+	inflight bool
+	pending  bool
+}
+
+// trigger schedules a save. If a save is already in-flight, marks
+// pending and returns. If none in-flight, starts a goroutine that
+// runs save and any queued pending saves.
+//
+// save is the actual save operation (parameterized for testability).
+// Errors are logged via slog and not returned — same fail-open
+// posture as the prior synchronous saveAfter.
+func (s *saveTask) trigger(save func() error) {
+	s.mu.Lock()
+	if s.inflight {
+		s.pending = true
+		s.mu.Unlock()
+		return
+	}
+	s.inflight = true
+	s.mu.Unlock()
+
+	go func() {
+		for {
+			if err := save(); err != nil {
+				slog.Warn("persist save", "err", err)
+			}
+			s.mu.Lock()
+			if !s.pending {
+				s.inflight = false
+				s.mu.Unlock()
+				return
+			}
+			s.pending = false
+			s.mu.Unlock()
+			// Loop: re-run save to capture changes that arrived
+			// while we were saving.
+		}
+	}()
 }
 
 // rehydrate enumerates persisted indexes and loads each into the
@@ -103,19 +171,38 @@ func (h *handlers) rehydrate(ctx context.Context) (int, error) {
 	return loaded, nil
 }
 
-// saveAfter is the post-write persistence hook. Logs-not-fatal:
-// in-memory state is the source of truth in flight; a failed save
-// gets re-attempted on the next mutation, and the operator log
-// shows the storaged outage.
+// saveAfter triggers a coalesced async persistence for the index.
+// In-memory state is the source of truth in flight; a failed save
+// re-runs on the next mutation, and the operator log shows the
+// storaged outage.
+//
+// Coalescing semantics (added 2026-05-01 after multitier_100k
+// follow-up): rapid concurrent writes collapse into at most two
+// MinIO PUTs per index (current + one catch-up), instead of one
+// per Add. See the saveTask docstring.
 func (h *handlers) saveAfter(idx *vectord.Index) {
 	if h.persist == nil {
 		return
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-	if err := h.persist.Save(ctx, idx); err != nil {
-		slog.Warn("persist save", "name", idx.Params().Name, "err", err)
+	name := idx.Params().Name
+	h.saversMu.Lock()
+	if h.savers == nil {
+		h.savers = make(map[string]*saveTask)
 	}
+	s, ok := h.savers[name]
+	if !ok {
+		s = &saveTask{}
+		h.savers[name] = s
+	}
+	h.saversMu.Unlock()
+	s.trigger(func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		if err := h.persist.Save(ctx, idx); err != nil {
+			return err
+		}
+		return nil
+	})
 }
 
 // deleteAfter mirrors saveAfter for the Delete path.

cmd/vectord/main_test.go

@@ -3,11 +3,15 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/go-chi/chi/v5"
 
@@ -417,3 +421,105 @@ func TestSearchK_DefaultsAndMax(t *testing.T) {
 		t.Errorf("maxK=%d unreasonably large", maxK)
 	}
 }
+
+// TestSaveTask_Coalesces locks the multitier_100k follow-up: a
+// burst of triggers must collapse into at most 2 actual saves
+// (the in-flight one + one catch-up). Without coalescing, every
+// trigger would yield a save and concurrent writers would
+// serialize on the index RLock during Encode (the original
+// 1-2.5s tail-latency cause).
+func TestSaveTask_Coalesces(t *testing.T) {
+	var (
+		s        saveTask
+		saveCnt  atomic.Int32
+		started  = make(chan struct{}, 1)
+		release  = make(chan struct{})
+	)
+	save := func() error {
+		// First save blocks until released so we can pile up
+		// triggers behind it. Subsequent saves return fast so the
+		// catch-up logic completes promptly.
+		n := saveCnt.Add(1)
+		if n == 1 {
+			started <- struct{}{}
+			<-release
+		}
+		return nil
+	}
+	// Trigger first save and wait for it to enter the blocked region.
+	s.trigger(save)
+	<-started
+	// Pile up triggers while the first is blocked. None of these
+	// should start their own goroutines — they should mark "pending".
+	for i := 0; i < 50; i++ {
+		s.trigger(save)
+	}
+	// Release the first save. The trigger logic should run ONE
+	// catch-up save for all 50 piled-up triggers, then return.
+	close(release)
+	// Wait for the goroutine to drain.
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		s.mu.Lock()
+		idle := !s.inflight && !s.pending
+		s.mu.Unlock()
+		if idle {
+			break
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	got := saveCnt.Load()
+	if got != 2 {
+		t.Errorf("save count = %d, want 2 (one in-flight + one catch-up)", got)
+	}
+}
+
+// TestSaveTask_RunsOnce — single trigger fires exactly one save.
+func TestSaveTask_RunsOnce(t *testing.T) {
+	var s saveTask
+	var n atomic.Int32
+	done := make(chan struct{})
+	s.trigger(func() error {
+		n.Add(1)
+		close(done)
+		return nil
+	})
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("trigger goroutine never ran")
+	}
+	// Wait briefly for the goroutine to mark inflight=false.
+	time.Sleep(20 * time.Millisecond)
+	if got := n.Load(); got != 1 {
+		t.Errorf("save count = %d, want 1", got)
+	}
+}
+
+// TestSaveTask_LogsSaveError — a save error doesn't break the
+// coalescing state machine; subsequent triggers still work.
+func TestSaveTask_LogsSaveError(t *testing.T) {
+	var s saveTask
+	var n atomic.Int32
+	wantErr := errors.New("boom")
+	var wg sync.WaitGroup
+	wg.Add(1)
+	s.trigger(func() error {
+		defer wg.Done()
+		n.Add(1)
+		return wantErr
+	})
+	wg.Wait()
+	// State must reset so the next trigger fires another save.
+	time.Sleep(20 * time.Millisecond)
+	wg.Add(1)
+	s.trigger(func() error {
+		defer wg.Done()
+		n.Add(1)
+		return nil
+	})
+	wg.Wait()
+	if got := n.Load(); got != 2 {
+		t.Errorf("save count = %d, want 2 (failure must not stall the task)", got)
+	}
+}