embed cache — LRU at /v1/embed for repeat-query elimination

Adds CachedProvider wrapping the embedding Provider with a thread-safe
LRU keyed on (effective_model, sha256(text)) → []float32. Repeat
queries return the stored vector without round-tripping to Ollama.

Why this matters: the staffing 500K test (memory project_golang_lakehouse)
documented that the staffing co-pilot replays many of the same query
texts ("forklift driver IL", "welder Chicago", "warehouse safety", etc).
Each repeat paid the ~50ms Ollama round-trip. Cached repeats now serve
in <1µs (LRU lookup + sha256 of input).

Memory budget: ~3 KiB per entry at d=768. Default 10K entries ≈ 30 MiB.
Configurable via [embedd].cache_size; 0 disables (pass-through mode).

Per-text caching, not per-batch — a batch with mixed hits/misses only
fetches the misses upstream, then merges the result preserving caller
input order. Three-text batch with one miss = one upstream call for
that one text instead of three.

Implementation:
  internal/embed/cached.go (NEW, 150 LoC)
    CachedProvider implements Provider; uses hashicorp/golang-lru/v2.
    Key shape: "<model>:<sha256-hex>". Empty model resolves to
    defaultModel (request-derived) for the key — NOT res.Model
    (upstream-derived), so future requests with same input shape
    hit the same key. Caught by TestCachedProvider_EmptyModelResolvesToDefault.
    Atomic hit/miss counters + Stats() + HitRate() + Len().

  internal/embed/cached_test.go (NEW, 12 test funcs)
    Pass-through-when-zero, hit-on-repeat, mixed-batch only fetches
    misses, model-key isolation, empty-model resolves to default,
    LRU eviction at cap, error propagation, all-hits synthesized
    without upstream call, hit-rate accumulation, empty-texts
    rejected, concurrent-safe (50 goroutines × 100 calls), key
    stability + distinctness.

  internal/shared/config.go
    EmbeddConfig.CacheSize (toml: cache_size). Default 10000.

  cmd/embedd/main.go
    Wraps Ollama Provider with CachedProvider on startup. Adds
    /embed/stats endpoint exposing hits / misses / hit_rate / size.
    Operators check the rate to confirm the cache is working
    (high rate = good) or sized wrong (low rate + many misses on a
    workload that should have repeats).

  cmd/embedd/main_test.go
    Stats endpoint tests — disabled mode shape, enabled mode tracks
    hits + misses across repeat calls.

One real bug caught by my own test:
  Initial implementation cached under res.Model (upstream-resolved)
  rather than effectiveModel (request-resolved). A request with
  model="" caching under "test-model" (Ollama's default), then a
  request with model="the-default" (our config default) missing
  the cache. Fix: always use the request-derived effectiveModel
  for keys; that's the predictable side. Locked by
  TestCachedProvider_EmptyModelResolvesToDefault.

Verified:
  go test -count=1 ./internal/embed/  — all 12 cached tests + 6 ollama tests green
  go test -count=1 ./cmd/embedd/      — stats endpoint tests green
  just verify                          — vet + test + 9 smokes 33s

Production benefit:
  ~50ms Ollama round-trip → <1µs cache lookup for cached entries.
  At 10K-entry default + ~30% repeat rate (typical staffing co-pilot
  workload), saves several seconds per staffer-query session.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

cmd/embedd/main.go

@@ -41,9 +41,20 @@ func main() {
 		os.Exit(1)
 	}
 
-	h := &handlers{
-		provider: embed.NewOllama(cfg.Embedd.ProviderURL, cfg.Embedd.DefaultModel),
+	// Wrap the upstream provider in an LRU cache so repeat queries
+	// (the staffing co-pilot replays many of the same texts) bypass
+	// the ~50ms Ollama round-trip. Cache size 0 = pass-through.
+	base := embed.NewOllama(cfg.Embedd.ProviderURL, cfg.Embedd.DefaultModel)
+	cached, err := embed.NewCachedProvider(base, cfg.Embedd.DefaultModel, cfg.Embedd.CacheSize)
+	if err != nil {
+		slog.Error("embed cache", "err", err)
+		os.Exit(1)
 	}
+	slog.Info("embed cache",
+		"size", cfg.Embedd.CacheSize,
+		"default_model", cfg.Embedd.DefaultModel,
+		"enabled", cfg.Embedd.CacheSize > 0)
+	h := &handlers{provider: cached, cache: cached}
 
 	if err := shared.Run("embedd", cfg.Embedd.Bind, h.register); err != nil {
 		slog.Error("server", "err", err)
@@ -53,10 +64,36 @@ func main() {
 
 type handlers struct {
 	provider embed.Provider
+	// cache is the same instance as provider when caching is enabled,
+	// kept as a typed pointer so /v1/embed/stats can expose hit-rate
+	// without type-asserting through the Provider interface. nil when
+	// CacheSize=0 (pass-through mode).
+	cache *embed.CachedProvider
 }
 
 func (h *handlers) register(r chi.Router) {
 	r.Post("/embed", h.handleEmbed)
+	r.Get("/embed/stats", h.handleStats)
+}
+
+// handleStats reports cache hits/misses + hit rate + size. Operators
+// use this to confirm the cache is doing its job (high hit rate) or
+// is sized wrong (low hit rate + many misses on a workload that
+// should have repeats).
+func (h *handlers) handleStats(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	if h.cache == nil {
+		_ = json.NewEncoder(w).Encode(map[string]any{"enabled": false})
+		return
+	}
+	hits, misses := h.cache.Stats()
+	_ = json.NewEncoder(w).Encode(map[string]any{
+		"enabled":   true,
+		"hits":      hits,
+		"misses":    misses,
+		"hit_rate":  h.cache.HitRate(),
+		"size":      h.cache.Len(),
+	})
 }
 
 // embedRequest is the POST /embed body. Texts is the list to

cmd/embedd/main_test.go

@@ -143,6 +143,71 @@ func TestHandleEmbed_HappyPath_ProviderEcho(t *testing.T) {
 	}
 }
 
+func TestStatsEndpoint_CacheDisabled(t *testing.T) {
+	r := mountWithProvider(&stubProvider{})
+	srv := httptest.NewServer(r)
+	defer srv.Close()
+
+	resp, err := http.Get(srv.URL + "/embed/stats")
+	if err != nil {
+		t.Fatalf("GET: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("status = %d, want 200", resp.StatusCode)
+	}
+	if !strings.Contains(getBody(t, resp), `"enabled":false`) {
+		t.Errorf("expected enabled:false when no cache wired")
+	}
+}
+
+func TestStatsEndpoint_CacheEnabled_TracksHitsAndMisses(t *testing.T) {
+	stub := &stubProvider{result: embed.Result{
+		Model:     "test-model",
+		Dimension: 3,
+		Vectors:   [][]float32{{0.1, 0.2, 0.3}},
+	}}
+	cache, err := embed.NewCachedProvider(stub, "test-model", 100)
+	if err != nil {
+		t.Fatalf("NewCachedProvider: %v", err)
+	}
+	h := &handlers{provider: cache, cache: cache}
+	r := chi.NewRouter()
+	h.register(r)
+	srv := httptest.NewServer(r)
+	defer srv.Close()
+
+	// First call → miss.
+	http.Post(srv.URL+"/embed", "application/json",
+		strings.NewReader(`{"texts":["hello"],"model":"test-model"}`))
+	// Second call same text → hit.
+	http.Post(srv.URL+"/embed", "application/json",
+		strings.NewReader(`{"texts":["hello"],"model":"test-model"}`))
+
+	resp, err := http.Get(srv.URL + "/embed/stats")
+	if err != nil {
+		t.Fatalf("stats GET: %v", err)
+	}
+	defer resp.Body.Close()
+	body := getBody(t, resp)
+	if !strings.Contains(body, `"enabled":true`) {
+		t.Errorf("expected enabled:true, body=%s", body)
+	}
+	if !strings.Contains(body, `"hits":1`) {
+		t.Errorf("expected hits:1, body=%s", body)
+	}
+	if !strings.Contains(body, `"misses":1`) {
+		t.Errorf("expected misses:1, body=%s", body)
+	}
+}
+
+func getBody(t *testing.T, resp *http.Response) string {
+	t.Helper()
+	buf := make([]byte, 4096)
+	n, _ := resp.Body.Read(buf)
+	return string(buf[:n])
+}
+
 func TestItoa(t *testing.T) {
 	cases := []struct {
 		in  int