ADR-004 + internal/pathway: Mem0 versioned trace substrate

Closes Sprint 2 design-bar work (audit reports/scrum/sprint-backlog.md): S2.1 — ADR-004 documents the pathway-memory data model S2.2 — pathway port lands with deterministic fixture corpus and full test coverage on day one S2.3 — retired traces are excluded from retrieval (test passes; would fail without the filter) Mem0-style operations: Add / AddIdempotent / Update / Revise / Retire / Get / History / Search. Each operation is a method on Store; persistence is JSONL append-only with corruption recovery on Replay. internal/pathway/types.go Trace + event + SearchFilter + sentinel errors internal/pathway/store.go in-memory state + RWMutex + ops internal/pathway/persistor.go JSONL append-only log with replay internal/pathway/store_test.go 20 test funcs covering all 7 Sprint 2 claim rows + concurrency internal/pathway/persistor_test.go 6 test funcs covering missing- file, corruption recovery, long-line handling, parent-dir auto-create, apply-error skip behavior Sprint 2 claim coverage row-by-row: ADD TestAdd_AssignsUIDAndTimestamps + TestAdd_RejectsInvalidJSON UPDATE TestUpdate_ReplacesContentSameUID + Update_MissingUID_Errors REVISE TestRevise_LinksToPredecessorViaHistory + TestRevise_PredecessorMissing_Errors + TestRevise_ChainOfThree_BackwardWalk RETIRE TestRetire_ExcludedFromSearch + TestRetire_StillAccessibleViaGet + TestRetire_StillAccessibleViaHistory HISTORY/cycle TestHistory_CycleDetected (injected via internal map), TestHistory_PredecessorMissing_TruncatesChain, TestHistory_UnknownUID_ErrorsClean REPLAY/dup TestAddIdempotent_IncrementsReplayCount (locks the "replay preserves original content" rule per ADR-004) CORRUPTION TestPersistor_CorruptedLines_Skipped + TestPersistor_ApplyError_Skipped ROUND-TRIP TestPersistor_RoundTrip locks the full Save → fresh Store → Load → Stats-match contract Two real bugs caught during testing: - Add returned the same *Trace stored in the map, so callers holding a reference saw later mutations. Fixed: clone before return (matches Get's contract). Same fix in AddIdempotent + Revise. - Test typo: {"v":different} isn't valid JSON; AddIdempotent's json.Valid rejected it as ErrInvalidContent. Test fixed to use {"v":"different"}; the validation behavior is correct. Skipped this commit (next): - cmd/pathwayd HTTP binary - gateway routing for /v1/pathway/* - end-to-end smoke These add the wire surface; the substrate ships first so the wire layer can be a pure proxy in the next commit. Verified: go test -count=1 ./internal/pathway/ — 26 tests green just verify — vet + test + 9 smokes 34s Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-29 07:23:30 -05:00
parent ad1670d36a
commit 2a6234ff82
6 changed files with 1302 additions and 3 deletions
@@ -0,0 +1,184 @@
+package pathway
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// persistor_test covers the corruption-recovery contract per
+// Sprint 2 row 7: malformed JSONL lines must not halt replay.
+
+func TestPersistor_MissingFileIsNotError(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "nonexistent.jsonl")
+	p, err := NewPersistor(path)
+	if err != nil {
+		t.Fatalf("NewPersistor on missing file should not error, got %v", err)
+	}
+	n, err := p.Replay(func(event) error { return nil })
+	if err != nil {
+		t.Errorf("Replay on missing file should be 0,nil; got %d, %v", n, err)
+	}
+	if n != 0 {
+		t.Errorf("Replay on missing file replayed %d events, want 0", n)
+	}
+}
+
+func TestPersistor_AppendThenReplay(t *testing.T) {
+	p := mustPersistor(t)
+
+	if err := p.Append(event{Op: opAdd, Trace: &Trace{UID: "A", Content: json.RawMessage(`{}`)}}); err != nil {
+		t.Fatalf("Append: %v", err)
+	}
+	if err := p.Append(event{Op: opAdd, Trace: &Trace{UID: "B", Content: json.RawMessage(`{}`)}}); err != nil {
+		t.Fatalf("Append: %v", err)
+	}
+
+	var seen []string
+	n, err := p.Replay(func(e event) error {
+		if e.Trace != nil {
+			seen = append(seen, e.Trace.UID)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Replay: %v", err)
+	}
+	if n != 2 {
+		t.Errorf("Replay applied %d events, want 2", n)
+	}
+	if len(seen) != 2 || seen[0] != "A" || seen[1] != "B" {
+		t.Errorf("seen = %v, want [A B]", seen)
+	}
+}
+
+func TestPersistor_CorruptedLines_Skipped(t *testing.T) {
+	p := mustPersistor(t)
+
+	// Mix of valid and corrupted lines.
+	good1 := mustMarshal(t, event{Op: opAdd, Trace: &Trace{UID: "A", Content: json.RawMessage(`{}`)}})
+	bad := []byte(`{this is not json}`)
+	good2 := mustMarshal(t, event{Op: opAdd, Trace: &Trace{UID: "B", Content: json.RawMessage(`{}`)}})
+	emptyLine := []byte(``)
+	good3 := mustMarshal(t, event{Op: opAdd, Trace: &Trace{UID: "C", Content: json.RawMessage(`{}`)}})
+
+	contents := []byte{}
+	for _, line := range [][]byte{good1, bad, good2, emptyLine, good3} {
+		contents = append(contents, line...)
+		contents = append(contents, '\n')
+	}
+	if err := os.WriteFile(p.Path(), contents, 0o644); err != nil {
+		t.Fatalf("write file: %v", err)
+	}
+
+	var applied []string
+	n, err := p.Replay(func(e event) error {
+		if e.Trace != nil {
+			applied = append(applied, e.Trace.UID)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Replay: %v", err)
+	}
+	// 3 valid + 1 bad + 1 empty (skipped silently) = 3 applied.
+	if n != 3 {
+		t.Errorf("Replay applied %d, want 3 (1 corrupt line skipped)", n)
+	}
+	if len(applied) != 3 || applied[0] != "A" || applied[1] != "B" || applied[2] != "C" {
+		t.Errorf("applied = %v, want [A B C]", applied)
+	}
+}
+
+func TestPersistor_ApplyError_Skipped(t *testing.T) {
+	// If the apply function returns error for an event, replay
+	// should keep going (the error is logged, not raised).
+	p := mustPersistor(t)
+	_ = p.Append(event{Op: opAdd, Trace: &Trace{UID: "A", Content: json.RawMessage(`{}`)}})
+	_ = p.Append(event{Op: opAdd, Trace: &Trace{UID: "B", Content: json.RawMessage(`{}`)}})
+	_ = p.Append(event{Op: opAdd, Trace: &Trace{UID: "C", Content: json.RawMessage(`{}`)}})
+
+	count := 0
+	n, err := p.Replay(func(e event) error {
+		if e.Trace != nil && e.Trace.UID == "B" {
+			return errors.New("simulated apply error on B")
+		}
+		count++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Replay: %v", err)
+	}
+	if n != 2 || count != 2 {
+		t.Errorf("Replay applied %d (callback called %d), want 2 each (B's error skipped)", n, count)
+	}
+}
+
+func TestPersistor_NewPersistor_EmptyPath_Errors(t *testing.T) {
+	_, err := NewPersistor("")
+	if err == nil {
+		t.Error("NewPersistor with empty path should error")
+	}
+}
+
+func TestPersistor_CreatesParentDir(t *testing.T) {
+	dir := t.TempDir()
+	nested := filepath.Join(dir, "nested", "deep", "pathway.jsonl")
+	p, err := NewPersistor(nested)
+	if err != nil {
+		t.Fatalf("NewPersistor: %v", err)
+	}
+	if err := p.Append(event{Op: opAdd, Trace: &Trace{UID: "A", Content: json.RawMessage(`{}`)}}); err != nil {
+		t.Fatalf("Append after creating nested dir: %v", err)
+	}
+}
+
+func TestPersistor_LongLine_HandlesUpTo1MiB(t *testing.T) {
+	p := mustPersistor(t)
+
+	// Build a content blob ~750 KiB so the JSON line is ~800 KiB
+	// (under the 1 MiB scanner cap).
+	blob := strings.Repeat("x", 750*1024)
+	bigContent, _ := json.Marshal(map[string]string{"data": blob})
+	tr := &Trace{UID: "BIG", Content: bigContent}
+	if err := p.Append(event{Op: opAdd, Trace: tr}); err != nil {
+		t.Fatalf("Append big trace: %v", err)
+	}
+
+	count := 0
+	n, _ := p.Replay(func(e event) error {
+		if e.Trace != nil && e.Trace.UID == "BIG" {
+			count++
+		}
+		return nil
+	})
+	if n != 1 || count != 1 {
+		t.Errorf("big-line replay: got %d events / %d matches, want 1 each", n, count)
+	}
+}
+
+// ── helpers ──
+
+func mustPersistor(t *testing.T) *Persistor {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.jsonl")
+	p, err := NewPersistor(path)
+	if err != nil {
+		t.Fatalf("NewPersistor: %v", err)
+	}
+	return p
+}
+
+func mustMarshal(t *testing.T, e event) []byte {
+	t.Helper()
+	b, err := json.Marshal(e)
+	if err != nil {
+		t.Fatalf("marshal: %v", err)
+	}
+	return b
+}