checkpoint: matrix-agent-validated (2026-04-25)

Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.

WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.

WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
  briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
    * UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
    * REVISE: chains versions, parent.superseded_at + superseded_by stamped
    * RETIRE: marks specific trace retired with reason, excluded from retrieval
    * HISTORY: walks chain root→tip, cycle-safe

KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces

Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

proto/lakehouse.proto

@@ -0,0 +1,166 @@
+syntax = "proto3";
+package lakehouse;
+
+// --- Catalog Service ---
+
+service CatalogService {
+    rpc CreateDataset(CreateDatasetRequest) returns (DatasetResponse);
+    rpc GetDataset(GetDatasetRequest) returns (DatasetResponse);
+    rpc GetDatasetByName(GetDatasetByNameRequest) returns (DatasetResponse);
+    rpc ListDatasets(ListDatasetsRequest) returns (ListDatasetsResponse);
+}
+
+message CreateDatasetRequest {
+    string name = 1;
+    string schema_fingerprint = 2;
+    repeated ObjectRef objects = 3;
+}
+
+message GetDatasetRequest {
+    string id = 1;
+}
+
+message GetDatasetByNameRequest {
+    string name = 1;
+}
+
+message ListDatasetsRequest {}
+
+message ListDatasetsResponse {
+    repeated DatasetResponse datasets = 1;
+}
+
+message DatasetResponse {
+    string id = 1;
+    string name = 2;
+    string schema_fingerprint = 3;
+    repeated ObjectRef objects = 4;
+    string created_at = 5;
+    string updated_at = 6;
+}
+
+message ObjectRef {
+    string bucket = 1;
+    string key = 2;
+    uint64 size_bytes = 3;
+    string created_at = 4;
+}
+
+// --- Query Service ---
+
+service QueryService {
+    rpc ExecuteQuery(QueryRequest) returns (QueryResponse);
+}
+
+message QueryRequest {
+    string sql = 1;
+}
+
+message QueryResponse {
+    repeated ColumnInfo columns = 1;
+    bytes rows_json = 2;  // JSON-encoded rows
+    uint64 row_count = 3;
+}
+
+message ColumnInfo {
+    string name = 1;
+    string data_type = 2;
+}
+
+// --- Storage Service ---
+
+service StorageService {
+    rpc PutObject(PutObjectRequest) returns (PutObjectResponse);
+    rpc GetObject(GetObjectRequest) returns (GetObjectResponse);
+    rpc DeleteObject(DeleteObjectRequest) returns (DeleteObjectResponse);
+    rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse);
+}
+
+message PutObjectRequest {
+    string key = 1;
+    bytes data = 2;
+}
+
+message PutObjectResponse {
+    string key = 1;
+}
+
+message GetObjectRequest {
+    string key = 1;
+}
+
+message GetObjectResponse {
+    bytes data = 1;
+}
+
+message DeleteObjectRequest {
+    string key = 1;
+}
+
+message DeleteObjectResponse {
+    string key = 1;
+}
+
+message ListObjectsRequest {
+    string prefix = 1;
+}
+
+message ListObjectsResponse {
+    repeated string keys = 1;
+}
+
+// --- AI Service ---
+
+service AiService {
+    rpc Embed(EmbedRequest) returns (EmbedResponse);
+    rpc Generate(GenerateRequest) returns (GenerateResponse);
+    rpc Rerank(RerankRequest) returns (RerankResponse);
+}
+
+message EmbedRequest {
+    repeated string texts = 1;
+    string model = 2;
+}
+
+message EmbedResponse {
+    repeated Embedding embeddings = 1;
+    string model = 2;
+    uint32 dimensions = 3;
+}
+
+message Embedding {
+    repeated double values = 1;
+}
+
+message GenerateRequest {
+    string prompt = 1;
+    string model = 2;
+    string system = 3;
+    double temperature = 4;
+    uint32 max_tokens = 5;
+}
+
+message GenerateResponse {
+    string text = 1;
+    string model = 2;
+    uint64 tokens_evaluated = 3;
+    uint64 tokens_generated = 4;
+}
+
+message RerankRequest {
+    string query = 1;
+    repeated string documents = 2;
+    string model = 3;
+    uint32 top_k = 4;
+}
+
+message RerankResponse {
+    repeated ScoredDocument results = 1;
+    string model = 2;
+}
+
+message ScoredDocument {
+    uint32 index = 1;
+    string text = 2;
+    double score = 3;
+}