checkpoint: matrix-agent-validated (2026-04-25)

Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.

WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.

WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
  briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
    * UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
    * REVISE: chains versions, parent.superseded_at + superseded_by stamped
    * RETIRE: marks specific trace retired with reason, excluded from retrieval
    * HISTORY: walks chain root→tip, cycle-safe

KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces

Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/shared/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "shared"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+uuid = { workspace = true }
+chrono = { workspace = true }
+arrow = { workspace = true }
+parquet = { workspace = true }
+bytes = { workspace = true }
+sha2 = { workspace = true }
+toml = { workspace = true }
+tracing = { workspace = true }
+tokio = { workspace = true }
+async-trait = "0.1"

crates/shared/src/arrow_helpers.rs

@@ -0,0 +1,83 @@
+use arrow::array::RecordBatch;
+use arrow::datatypes::SchemaRef;
+use bytes::Bytes;
+use parquet::arrow::ArrowWriter;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use sha2::{Digest, Sha256};
+
+use crate::types::SchemaFingerprint;
+
+/// Write a RecordBatch to Parquet bytes.
+pub fn record_batch_to_parquet(batch: &RecordBatch) -> Result<Bytes, String> {
+    let mut buf = Vec::new();
+    let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), None)
+        .map_err(|e| e.to_string())?;
+    writer.write(batch).map_err(|e| e.to_string())?;
+    writer.close().map_err(|e| e.to_string())?;
+    Ok(Bytes::from(buf))
+}
+
+/// Read Parquet bytes into a Vec of RecordBatches.
+pub fn parquet_to_record_batches(data: &[u8]) -> Result<(SchemaRef, Vec<RecordBatch>), String> {
+    let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::copy_from_slice(data))
+        .map_err(|e| e.to_string())?;
+    let schema = builder.schema().clone();
+    let reader = builder.build().map_err(|e| e.to_string())?;
+    let batches: Vec<RecordBatch> = reader.collect::<Result<Vec<_>, _>>().map_err(|e| e.to_string())?;
+    Ok((schema, batches))
+}
+
+/// Compute a deterministic fingerprint from an Arrow schema.
+pub fn fingerprint_schema(schema: &SchemaRef) -> SchemaFingerprint {
+    let mut hasher = Sha256::new();
+    for field in schema.fields() {
+        hasher.update(field.name().as_bytes());
+        hasher.update(field.data_type().to_string().as_bytes());
+        hasher.update(if field.is_nullable() { b"1" } else { b"0" });
+    }
+    let hash = hasher.finalize();
+    SchemaFingerprint(format!("{hash:x}"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow::array::{Float64Array, Int32Array, StringArray};
+    use arrow::datatypes::{DataType, Field, Schema};
+    use std::sync::Arc;
+
+    fn sample_batch() -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new("score", DataType::Float64, true),
+        ]));
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["alice", "bob", "carol"])),
+                Arc::new(Float64Array::from(vec![9.5, 8.2, 7.8])),
+            ],
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn roundtrip_parquet() {
+        let batch = sample_batch();
+        let parquet_bytes = record_batch_to_parquet(&batch).unwrap();
+        let (schema, batches) = parquet_to_record_batches(&parquet_bytes).unwrap();
+        assert_eq!(schema.fields().len(), 3);
+        assert_eq!(batches.len(), 1);
+        assert_eq!(batches[0].num_rows(), 3);
+    }
+
+    #[test]
+    fn schema_fingerprint_deterministic() {
+        let batch = sample_batch();
+        let fp1 = fingerprint_schema(&batch.schema());
+        let fp2 = fingerprint_schema(&batch.schema());
+        assert_eq!(fp1, fp2);
+    }
+}

crates/shared/src/config.rs

@@ -0,0 +1,203 @@
+use serde::Deserialize;
+use std::path::Path;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Config {
+    pub gateway: GatewayConfig,
+    pub storage: StorageConfig,
+    #[serde(default)]
+    pub catalog: CatalogConfig,
+    #[serde(default)]
+    pub query: QueryConfig,
+    pub sidecar: SidecarConfig,
+    #[serde(default)]
+    pub ai: AiConfig,
+    #[serde(default)]
+    pub auth: AuthConfig,
+    #[serde(default)]
+    pub observability: ObservabilityConfig,
+    #[serde(default)]
+    pub agent: AgentSettings,
+}
+
+/// Phase 16.2 — background autotune agent settings.
+///
+/// Duplicated from `vectord::agent::AgentConfig` because `shared` can't
+/// depend on `vectord` (vectord already depends on shared). The gateway
+/// copies these into the vectord config at startup.
+#[derive(Debug, Clone, Deserialize)]
+pub struct AgentSettings {
+    #[serde(default)]
+    pub enabled: bool,
+    #[serde(default = "default_cycle_interval_secs")]
+    pub cycle_interval_secs: u64,
+    #[serde(default = "default_cooldown_secs")]
+    pub cooldown_between_trials_secs: u64,
+    #[serde(default = "default_min_recall")]
+    pub min_recall: f32,
+    #[serde(default = "default_max_trials_per_hour")]
+    pub max_trials_per_hour: u32,
+}
+
+impl Default for AgentSettings {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            cycle_interval_secs: default_cycle_interval_secs(),
+            cooldown_between_trials_secs: default_cooldown_secs(),
+            min_recall: default_min_recall(),
+            max_trials_per_hour: default_max_trials_per_hour(),
+        }
+    }
+}
+
+fn default_cycle_interval_secs() -> u64 { 60 }
+fn default_cooldown_secs() -> u64 { 30 }
+fn default_min_recall() -> f32 { 0.9 }
+fn default_max_trials_per_hour() -> u32 { 30 }
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct GatewayConfig {
+    #[serde(default = "default_host")]
+    pub host: String,
+    #[serde(default = "default_gateway_port")]
+    pub port: u16,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct StorageConfig {
+    /// Legacy single-backend root. If `buckets` is empty, this is used to
+    /// create an implicit `primary` bucket at this path — preserves the
+    /// pre-federation config shape.
+    #[serde(default = "default_storage_root")]
+    pub root: String,
+
+    /// Where profile buckets are rooted when auto-provisioned.
+    #[serde(default = "default_profile_root")]
+    pub profile_root: String,
+
+    /// Name of the bucket used for read fallback when a target bucket is
+    /// unreachable. If `None`, no fallback — reads fail hard.
+    #[serde(default)]
+    pub rescue_bucket: Option<String>,
+
+    /// Explicitly configured buckets. Empty = backward-compat single-bucket
+    /// mode driven by `root`.
+    #[serde(default)]
+    pub buckets: Vec<BucketConfig>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BucketConfig {
+    pub name: String,
+    pub backend: String,           // "local" | "s3"
+    /// Local filesystem root (for backend = "local")
+    pub root: Option<String>,
+    /// S3 bucket name (for backend = "s3")
+    pub bucket: Option<String>,
+    pub region: Option<String>,
+    pub endpoint: Option<String>,
+    /// Handle for the secrets provider — never the literal credential.
+    pub secret_ref: Option<String>,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct CatalogConfig {
+    #[serde(default = "default_manifest_prefix")]
+    pub manifest_prefix: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct QueryConfig {
+    pub max_rows_per_query: Option<usize>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct SidecarConfig {
+    #[serde(default = "default_sidecar_url")]
+    pub url: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct AiConfig {
+    #[serde(default = "default_embed_model")]
+    pub embed_model: String,
+    #[serde(default = "default_gen_model")]
+    pub gen_model: String,
+    #[serde(default = "default_rerank_model")]
+    pub rerank_model: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct AuthConfig {
+    #[serde(default)]
+    pub enabled: bool,
+    pub api_key: Option<String>,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct ObservabilityConfig {
+    #[serde(default = "default_exporter")]
+    pub exporter: String,
+    #[serde(default = "default_service_name")]
+    pub service_name: String,
+}
+
+// Defaults
+fn default_host() -> String { "0.0.0.0".to_string() }
+fn default_gateway_port() -> u16 { 3100 }
+fn default_storage_root() -> String { "./data".to_string() }
+fn default_profile_root() -> String { "./data/_profiles".to_string() }
+fn default_manifest_prefix() -> String { "_catalog/manifests".to_string() }
+fn default_sidecar_url() -> String { "http://localhost:3200".to_string() }
+fn default_embed_model() -> String { "nomic-embed-text".to_string() }
+fn default_gen_model() -> String { "qwen2.5".to_string() }
+fn default_rerank_model() -> String { "qwen2.5".to_string() }
+fn default_exporter() -> String { "stdout".to_string() }
+fn default_service_name() -> String { "lakehouse".to_string() }
+
+impl Config {
+    pub fn load(path: &str) -> Result<Self, String> {
+        let path = Path::new(path);
+        if !path.exists() {
+            return Err(format!("config file not found: {}", path.display()));
+        }
+        let content = std::fs::read_to_string(path)
+            .map_err(|e| format!("failed to read config: {e}"))?;
+        toml::from_str(&content)
+            .map_err(|e| format!("failed to parse config: {e}"))
+    }
+
+    pub fn load_or_default() -> Self {
+        // Try lakehouse.toml in current dir, then /etc/lakehouse/lakehouse.toml
+        for path in &["lakehouse.toml", "/etc/lakehouse/lakehouse.toml"] {
+            if let Ok(config) = Self::load(path) {
+                tracing::info!("loaded config from {path}");
+                return config;
+            }
+        }
+        tracing::warn!("no config file found, using defaults");
+        Self::default()
+    }
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            gateway: GatewayConfig { host: default_host(), port: default_gateway_port() },
+            storage: StorageConfig {
+                root: default_storage_root(),
+                profile_root: default_profile_root(),
+                rescue_bucket: None,
+                buckets: Vec::new(),
+            },
+            catalog: CatalogConfig::default(),
+            query: QueryConfig::default(),
+            sidecar: SidecarConfig { url: default_sidecar_url() },
+            ai: AiConfig::default(),
+            auth: AuthConfig::default(),
+            observability: ObservabilityConfig::default(),
+            agent: AgentSettings::default(),
+        }
+    }
+}

crates/shared/src/errors.rs

@@ -0,0 +1,35 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum LakehouseError {
+    #[error("dataset not found: {0}")]
+    DatasetNotFound(String),
+
+    #[error("object not found: {bucket}/{key}")]
+    ObjectNotFound { bucket: String, key: String },
+
+    #[error("storage error: {0}")]
+    Storage(String),
+
+    #[error("catalog error: {0}")]
+    Catalog(String),
+
+    #[error("query error: {0}")]
+    Query(String),
+
+    #[error("ai bridge error: {0}")]
+    AiBridge(String),
+
+    #[error("serialization error: {0}")]
+    Serialization(String),
+}
+
+impl LakehouseError {
+    pub fn status_code(&self) -> u16 {
+        match self {
+            Self::DatasetNotFound(_) | Self::ObjectNotFound { .. } => 404,
+            Self::Storage(_) | Self::Catalog(_) | Self::Query(_) | Self::AiBridge(_) => 500,
+            Self::Serialization(_) => 400,
+        }
+    }
+}

crates/shared/src/lib.rs

@@ -0,0 +1,8 @@
+pub mod types;
+pub mod errors;
+pub mod arrow_helpers;
+pub mod config;
+pub mod pii;
+pub mod secrets;
+pub mod model_matrix;
+pub mod profiles;

crates/shared/src/model_matrix.rs

@@ -0,0 +1,69 @@
+//! Per-model token accounting. Entry point for the ModelMatrix work
+//! the aibridge `context::estimate_tokens` deprecation has been pointing
+//! at. Starts minimal — just `estimate_tokens` — so call sites can
+//! migrate off the deprecated helper. Extend with per-model context
+//! windows, max_tokens defaults, provider hints, etc. as we move the
+//! rest of `aibridge::context::known_windows` over.
+
+/// Namespace for per-model token + context accounting. Methods are
+/// associated functions — no instance required — because the underlying
+/// estimates are deterministic and stateless.
+pub struct ModelMatrix;
+
+impl ModelMatrix {
+    /// Rough token count — char count divided by 4, rounded up. This
+    /// is the same heuristic OpenAI's cookbook uses for English text;
+    /// it's within ±15% of BPE tokenizers for code + prose and doesn't
+    /// require a tokenizer lookup. Good enough for budget math where
+    /// the goal is "don't blow the context window" rather than exact
+    /// billing.
+    ///
+    /// Moved from `aibridge::context::estimate_tokens` (still there with
+    /// a `#[deprecated]` pointer — callers should migrate here). Empty
+    /// string → 0; one char → 1 (ceiling of 1/4 = 1).
+    pub fn estimate_tokens(text: &str) -> usize {
+        (text.chars().count() + 3) / 4
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ModelMatrix;
+
+    #[test]
+    fn empty_string_is_zero_tokens() {
+        assert_eq!(ModelMatrix::estimate_tokens(""), 0);
+    }
+
+    #[test]
+    fn three_chars_is_one_token() {
+        // 3 → ceil(3/4) = 1. Matches the deprecated helper's behavior
+        // so the migration is a drop-in replacement.
+        assert_eq!(ModelMatrix::estimate_tokens("abc"), 1);
+    }
+
+    #[test]
+    fn four_chars_is_one_token() {
+        assert_eq!(ModelMatrix::estimate_tokens("abcd"), 1);
+    }
+
+    #[test]
+    fn five_chars_is_two_tokens() {
+        assert_eq!(ModelMatrix::estimate_tokens("abcde"), 2);
+    }
+
+    #[test]
+    fn counts_chars_not_bytes() {
+        // Multi-byte UTF-8 chars count as 1 char each — important for
+        // prompts with emoji or non-ASCII text. "héllo" is 5 chars
+        // (5 unicode scalars) → ceil(5/4) = 2 tokens, same as "hello".
+        assert_eq!(ModelMatrix::estimate_tokens("héllo"), 2);
+        assert_eq!(ModelMatrix::estimate_tokens("📚📚📚📚"), 1);  // 4 chars
+    }
+
+    #[test]
+    fn large_text_scales_linearly() {
+        assert_eq!(ModelMatrix::estimate_tokens(&"x".repeat(400)), 100);
+        assert_eq!(ModelMatrix::estimate_tokens(&"x".repeat(401)), 101);
+    }
+}

crates/shared/src/pii.rs

@@ -0,0 +1,116 @@
+/// Auto-detect PII columns by name patterns.
+/// Conservative: flags likely PII, doesn't miss obvious cases.
+
+use crate::types::Sensitivity;
+
+/// Check if a column name suggests PII content.
+pub fn detect_sensitivity(column_name: &str) -> Option<Sensitivity> {
+    let lower = column_name.to_lowercase();
+
+    // Direct PII identifiers
+    if matches!(lower.as_str(),
+        "ssn" | "social_security" | "social_security_number" |
+        "sin" | "national_id" | "passport" | "passport_number" |
+        "drivers_license" | "driver_license" | "dl_number"
+    ) {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Names
+    if lower.contains("first_name") || lower.contains("last_name") ||
+       lower.contains("full_name") || lower.contains("middle_name") ||
+       lower == "name" || lower == "fname" || lower == "lname" {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Contact info
+    if lower.contains("email") || lower.contains("e_mail") ||
+       lower.contains("phone") || lower.contains("mobile") || lower.contains("cell") ||
+       lower.contains("fax") || lower == "tel" || lower == "telephone" {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Address
+    if lower.contains("address") || lower.contains("street") ||
+       (lower.contains("zip") && !lower.contains("unzip")) ||
+       lower == "postal_code" || lower == "postcode" {
+        return Some(Sensitivity::Pii);
+    }
+
+    // Financial
+    if lower.contains("salary") || lower.contains("wage") ||
+       lower.contains("pay_rate") || lower.contains("bill_rate") ||
+       lower.contains("compensation") || lower.contains("revenue") ||
+       lower.contains("bank_account") || lower.contains("routing_number") ||
+       lower.contains("credit_card") {
+        return Some(Sensitivity::Financial);
+    }
+
+    // Health
+    if lower.contains("diagnosis") || lower.contains("medication") ||
+       lower.contains("medical") || lower.contains("health") ||
+       lower.contains("patient_id") || lower.contains("mrn") {
+        return Some(Sensitivity::Phi);
+    }
+
+    // Date of birth
+    if lower == "dob" || lower == "date_of_birth" || lower == "birthdate" || lower == "birth_date" {
+        return Some(Sensitivity::Pii);
+    }
+
+    None
+}
+
+/// Classify all columns and return the highest sensitivity found.
+pub fn detect_dataset_sensitivity(column_names: &[&str]) -> Option<Sensitivity> {
+    let mut highest: Option<Sensitivity> = None;
+
+    for name in column_names {
+        if let Some(sens) = detect_sensitivity(name) {
+            highest = Some(match (&highest, &sens) {
+                (None, s) => s.clone(),
+                (Some(Sensitivity::Public), s) => s.clone(),
+                (Some(Sensitivity::Internal), s) if !matches!(s, Sensitivity::Public | Sensitivity::Internal) => s.clone(),
+                (Some(Sensitivity::Financial), Sensitivity::Pii | Sensitivity::Phi) => sens.clone(),
+                (Some(Sensitivity::Pii), Sensitivity::Phi) => Sensitivity::Phi,
+                (existing, _) => existing.clone().unwrap(),
+            });
+        }
+    }
+
+    highest
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn detects_email_as_pii() {
+        assert_eq!(detect_sensitivity("email"), Some(Sensitivity::Pii));
+        assert_eq!(detect_sensitivity("contact_email"), Some(Sensitivity::Pii));
+    }
+
+    #[test]
+    fn detects_salary_as_financial() {
+        assert_eq!(detect_sensitivity("salary"), Some(Sensitivity::Financial));
+        assert_eq!(detect_sensitivity("bill_rate"), Some(Sensitivity::Financial));
+    }
+
+    #[test]
+    fn detects_ssn() {
+        assert_eq!(detect_sensitivity("ssn"), Some(Sensitivity::Pii));
+    }
+
+    #[test]
+    fn non_sensitive_returns_none() {
+        assert_eq!(detect_sensitivity("status"), None);
+        assert_eq!(detect_sensitivity("created_at"), None);
+    }
+
+    #[test]
+    fn dataset_sensitivity_picks_highest() {
+        let cols = vec!["id", "name", "email", "status"];
+        assert_eq!(detect_dataset_sensitivity(&cols), Some(Sensitivity::Pii));
+    }
+}

crates/shared/src/profiles/execution.rs

@@ -0,0 +1,14 @@
+//! ExecutionProfile — the Phase 41 rename of Phase 17's ModelProfile.
+//!
+//! Carries what's needed to RUN inference: model tag, dataset bindings,
+//! HNSW config, embed model, bucket binding. Today this is a type
+//! alias over `crate::types::ModelProfile` — the PRD's
+//! "Backward compat: ModelProfile still loads, aliased to
+//! ExecutionProfile" line, honored literally.
+//!
+//! When the migration off the old name finishes, this file can either
+//! absorb the full struct definition or continue as an alias. Callers
+//! should reference `ExecutionProfile` going forward; `ModelProfile`
+//! stays exported from `types` for on-disk schema compat.
+
+pub use crate::types::ModelProfile as ExecutionProfile;

crates/shared/src/profiles/memory.rs

@@ -0,0 +1,38 @@
+//! MemoryProfile — how the agent's execution memory is kept.
+//!
+//! Phase 41 decomposition: the Phase 19 playbook_memory + Phase 26
+//! successful_playbooks + Phase 45 doc_refs all need per-profile
+//! tuning. Rather than bolt those onto ExecutionProfile, they live
+//! here so a "thin" execution profile can reuse a "fat" memory
+//! profile and vice versa.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct MemoryProfile {
+    pub id: String,
+    #[serde(default)]
+    pub description: String,
+    /// Phase 19: ceiling for playbook_memory boost on retrieval. 0
+    /// disables the boost entirely.
+    #[serde(default = "default_boost_ceiling")]
+    pub playbook_boost_ceiling: f32,
+    /// Phase 26: max history entries retained before rotation.
+    #[serde(default = "default_history_cap")]
+    pub history_cap: usize,
+    /// Phase 45: stale threshold for doc_refs before drift check
+    /// fires (hours).
+    #[serde(default = "default_stale_hours")]
+    pub doc_stale_hours: u32,
+    /// Phase 28: auto-retire playbooks that fail 3+ consecutive runs.
+    #[serde(default = "default_true")]
+    pub auto_retire_on_failure: bool,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+}
+
+fn default_boost_ceiling() -> f32 { 0.35 }
+fn default_history_cap() -> usize { 1000 }
+fn default_stale_hours() -> u32 { 168 } // one week
+fn default_true() -> bool { true }

crates/shared/src/profiles/mod.rs

@@ -0,0 +1,28 @@
+//! Phase 41 profile types.
+//!
+//! The existing `ModelProfile` (Phase 17) is aliased as
+//! `ExecutionProfile` here — it continues to carry the model +
+//! bindings + HNSW config needed to run inference. Three new profile
+//! types land alongside: `RetrievalProfile`, `MemoryProfile`,
+//! `ObserverProfile` — each owns a distinct slice of what used to be
+//! bundled.
+//!
+//! Backward-compat rule (PRD Phase 41): existing `ModelProfile` on
+//! disk continues to deserialize unchanged. New fields on the new
+//! profile types are `#[serde(default)]` so old payloads load with
+//! empty defaults.
+//!
+//! These are the canonical shapes — downstream code converts via
+//! `From<ModelProfile> for ExecutionProfile` (they're the same struct
+//! today, just named differently) and constructs the other three
+//! as needed.
+
+pub mod execution;
+pub mod retrieval;
+pub mod memory;
+pub mod observer;
+
+pub use execution::ExecutionProfile;
+pub use memory::MemoryProfile;
+pub use observer::ObserverProfile;
+pub use retrieval::RetrievalProfile;

crates/shared/src/profiles/observer.rs

@@ -0,0 +1,38 @@
+//! ObserverProfile — how loudly the observer logs this workload.
+//!
+//! Phase 41 decomposition: the observer's alert thresholds, escalation
+//! cadence, and log retention need per-workload tuning. Hot-path
+//! staffing workflows want aggressive alerting; batch backfills want
+//! quieter. This profile is read by mcp-server/observer.ts at
+//! activation-time.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ObserverProfile {
+    pub id: String,
+    #[serde(default)]
+    pub description: String,
+    /// How many consecutive failures trigger a cluster escalation to
+    /// LLM Team `/v1/chat` (qwen3-coder:480b).
+    #[serde(default = "default_failure_cluster_size")]
+    pub failure_cluster_size: u32,
+    /// Minimum seconds between alert emails for the same sig_hash.
+    /// Prevents alert storms during a regression.
+    #[serde(default = "default_alert_cooldown")]
+    pub alert_cooldown_secs: u32,
+    /// Observer ring buffer size. Older events fall off when full.
+    #[serde(default = "default_ring_size")]
+    pub ring_size: usize,
+    /// Whether to forward events to external Langfuse
+    /// (/v1/langfuse_trace). Off by default.
+    #[serde(default)]
+    pub forward_to_langfuse: bool,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+}
+
+fn default_failure_cluster_size() -> u32 { 3 }
+fn default_alert_cooldown() -> u32 { 300 } // 5 minutes
+fn default_ring_size() -> usize { 2000 }

crates/shared/src/profiles/retrieval.rs

@@ -0,0 +1,52 @@
+//! RetrievalProfile — what + how the agent reaches into memory.
+//!
+//! Phase 41 decomposition: the old ModelProfile bundled "what dataset
+//! can I read" (bound_datasets) AND "how do I rank results"
+//! (hnsw_config) with the model tag. Retrieval concerns split out here
+//! so a profile can swap its retrieval strategy without re-activating
+//! the model.
+//!
+//! Fields chosen for what's actually varied per-workload today:
+//!   - `top_k` / `rerank_top_k` — how many hits to fetch + rerank
+//!   - `freshness_cutoff_days` — Phase 45 doc-drift uses this
+//!   - `boost_playbook_memory` — Phase 19 meta-index feedback
+//!   - `enforce_sensitivity_gates` — Phase 13 access-control integration
+//!
+//! All fields are `#[serde(default)]` so loading a profile file that
+//! predates Phase 41 works without migration.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct RetrievalProfile {
+    /// Unique id — slug form, separate namespace from ExecutionProfile.
+    pub id: String,
+    /// Free-text operator description.
+    #[serde(default)]
+    pub description: String,
+    /// Default top-K for /vectors/search + /vectors/hybrid.
+    #[serde(default = "default_top_k")]
+    pub top_k: u32,
+    /// How many of the top-K to pass through the reranker. 0 disables
+    /// reranking for this profile.
+    #[serde(default = "default_rerank_top_k")]
+    pub rerank_top_k: u32,
+    /// Don't consider playbooks / docs older than this (days). 0 or
+    /// absent = no freshness filter.
+    #[serde(default)]
+    pub freshness_cutoff_days: u32,
+    /// Phase 19: boost workers/results by playbook_memory similarity.
+    #[serde(default)]
+    pub boost_playbook_memory: bool,
+    /// Phase 13: apply access-control masking on sensitive columns.
+    /// Default on — safety-first.
+    #[serde(default = "default_true")]
+    pub enforce_sensitivity_gates: bool,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+}
+
+fn default_top_k() -> u32 { 10 }
+fn default_rerank_top_k() -> u32 { 5 }
+fn default_true() -> bool { true }

crates/shared/src/secrets.rs

@@ -0,0 +1,144 @@
+/// Secrets layer for bucket credentials.
+///
+/// Credentials never appear in `lakehouse.toml`. Each bucket entry has an
+/// opaque `secret_ref` handle, resolved at startup by a `SecretsProvider`.
+///
+/// MVP ships `FileSecretsProvider` — reads a TOML file at a path given by
+/// `LAKEHOUSE_SECRETS` env var, defaulting to `/etc/lakehouse/secrets.toml`.
+/// The file should be root-owned, mode 0600, never in git.
+///
+/// Future providers (Vault, SOPS, OS keyring) implement the same trait.
+
+use async_trait::async_trait;
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub struct BucketCredentials {
+    pub access_key: String,
+    pub secret_key: String,
+}
+
+// Redact credentials when debug-printing — NEVER let these end up in logs.
+impl BucketCredentials {
+    pub fn redacted(&self) -> String {
+        let prefix: String = self.access_key.chars().take(4).collect();
+        format!("access_key={}... (redacted)", prefix)
+    }
+}
+
+#[async_trait]
+pub trait SecretsProvider: Send + Sync {
+    /// Resolve a handle to credentials. Returns Err if the handle is unknown.
+    async fn resolve(&self, handle: &str) -> Result<BucketCredentials, String>;
+
+    /// List known handles (for diagnostics only, never values).
+    async fn list_handles(&self) -> Vec<String>;
+}
+
+#[derive(Debug, Deserialize)]
+struct RawSecretEntry {
+    access_key: String,
+    secret_key: String,
+}
+
+/// File-backed secrets provider.
+pub struct FileSecretsProvider {
+    path: PathBuf,
+    cache: tokio::sync::RwLock<HashMap<String, BucketCredentials>>,
+}
+
+impl FileSecretsProvider {
+    pub fn new(path: impl Into<PathBuf>) -> Self {
+        Self {
+            path: path.into(),
+            cache: tokio::sync::RwLock::new(HashMap::new()),
+        }
+    }
+
+    /// Default path: env var `LAKEHOUSE_SECRETS` or `/etc/lakehouse/secrets.toml`.
+    pub fn from_env() -> Self {
+        let path = std::env::var("LAKEHOUSE_SECRETS")
+            .unwrap_or_else(|_| "/etc/lakehouse/secrets.toml".to_string());
+        Self::new(path)
+    }
+
+    /// Arc wrapper for convenient injection.
+    pub fn shared(path: impl Into<PathBuf>) -> Arc<dyn SecretsProvider> {
+        Arc::new(Self::new(path))
+    }
+
+    /// Load and populate the cache. Call once at startup.
+    pub async fn load(&self) -> Result<usize, String> {
+        if !self.path.exists() {
+            tracing::info!(
+                "secrets file {} not present — running with zero credentials configured",
+                self.path.display()
+            );
+            return Ok(0);
+        }
+        check_permissions(&self.path)?;
+        let content = std::fs::read_to_string(&self.path)
+            .map_err(|e| format!("read secrets file: {e}"))?;
+        let raw: HashMap<String, RawSecretEntry> = toml::from_str(&content)
+            .map_err(|e| format!("parse secrets file: {e}"))?;
+
+        let mut cache = self.cache.write().await;
+        for (handle, entry) in raw {
+            cache.insert(handle.clone(), BucketCredentials {
+                access_key: entry.access_key,
+                secret_key: entry.secret_key,
+            });
+        }
+        let count = cache.len();
+        tracing::info!("secrets: loaded {count} handles from {}", self.path.display());
+        Ok(count)
+    }
+}
+
+#[async_trait]
+impl SecretsProvider for FileSecretsProvider {
+    async fn resolve(&self, handle: &str) -> Result<BucketCredentials, String> {
+        let cache = self.cache.read().await;
+        cache
+            .get(handle)
+            .cloned()
+            .ok_or_else(|| format!("secret handle not found: {handle}"))
+    }
+
+    async fn list_handles(&self) -> Vec<String> {
+        self.cache.read().await.keys().cloned().collect()
+    }
+}
+
+/// Reject world-readable secrets files. Fail closed.
+fn check_permissions(path: &Path) -> Result<(), String> {
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let meta = std::fs::metadata(path)
+            .map_err(|e| format!("stat secrets file: {e}"))?;
+        let mode = meta.permissions().mode() & 0o777;
+        if mode & 0o044 != 0 {
+            return Err(format!(
+                "secrets file {} is group/world-readable (mode {:o}); chmod 600 to proceed",
+                path.display(), mode
+            ));
+        }
+    }
+    let _ = path;
+    Ok(())
+}
+
+/// A zero-handles provider for tests / purely-local setups.
+pub struct EmptySecretsProvider;
+
+#[async_trait]
+impl SecretsProvider for EmptySecretsProvider {
+    async fn resolve(&self, handle: &str) -> Result<BucketCredentials, String> {
+        Err(format!("no secrets provider configured (wanted handle '{handle}')"))
+    }
+    async fn list_handles(&self) -> Vec<String> { Vec::new() }
+}

crates/shared/src/types.rs

@@ -0,0 +1,366 @@
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+/// Unique identifier for a dataset in the catalog.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct DatasetId(pub Uuid);
+
+impl DatasetId {
+    pub fn new() -> Self {
+        Self(Uuid::new_v4())
+    }
+}
+
+impl std::fmt::Display for DatasetId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+/// Pointer to an object in storage. This is what the catalog stores — never raw data.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ObjectRef {
+    pub bucket: String,
+    pub key: String,
+    pub size_bytes: u64,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+}
+
+/// Schema fingerprint — deterministic hash of an Arrow schema.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct SchemaFingerprint(pub String);
+
+/// Sensitivity classification for a field or dataset.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Sensitivity {
+    Public,      // anyone can see
+    Internal,    // employees only
+    Pii,         // personally identifiable (name, email, phone, SSN)
+    Phi,         // protected health info (HIPAA)
+    Financial,   // compensation, billing, revenue
+}
+
+/// Column-level metadata.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ColumnMeta {
+    pub name: String,
+    pub data_type: String,
+    #[serde(default)]
+    pub sensitivity: Option<Sensitivity>,
+    #[serde(default)]
+    pub description: String,
+    #[serde(default)]
+    pub is_pii: bool,
+}
+
+/// Data lineage — where this dataset came from.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Lineage {
+    pub source_system: String,    // "csv_upload", "mysql_import", "api_ingest"
+    pub source_file: String,      // original filename or URI
+    pub ingest_job: String,       // job ID that created this dataset
+    pub ingest_timestamp: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub parent_datasets: Vec<String>, // derived from these datasets
+}
+
+/// Freshness contract — how often this data should be updated.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FreshnessContract {
+    pub expected_frequency: String, // "daily", "weekly", "hourly", "manual"
+    pub stale_after_hours: u32,     // alert if not updated within this many hours
+}
+
+/// Dataset manifest — the catalog's complete view of a dataset.
+/// All new fields use #[serde(default)] for backward compatibility with existing manifests.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DatasetManifest {
+    pub id: DatasetId,
+    pub name: String,
+    pub schema_fingerprint: SchemaFingerprint,
+    pub objects: Vec<ObjectRef>,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub updated_at: chrono::DateTime<chrono::Utc>,
+
+    // --- Rich metadata (Phase 10) ---
+
+    /// Human-readable description (auto-generated on ingest or manual)
+    #[serde(default)]
+    pub description: String,
+
+    /// Who owns this dataset
+    #[serde(default)]
+    pub owner: String,
+
+    /// Overall sensitivity classification
+    #[serde(default)]
+    pub sensitivity: Option<Sensitivity>,
+
+    /// Column-level metadata including sensitivity and descriptions
+    #[serde(default)]
+    pub columns: Vec<ColumnMeta>,
+
+    /// Where this data came from
+    #[serde(default)]
+    pub lineage: Option<Lineage>,
+
+    /// How fresh the data should be
+    #[serde(default)]
+    pub freshness: Option<FreshnessContract>,
+
+    /// Free-form tags for discovery
+    #[serde(default)]
+    pub tags: Vec<String>,
+
+    /// Row count (updated on ingest/compact)
+    #[serde(default)]
+    pub row_count: Option<u64>,
+
+    // --- Embedding freshness (Phase C) ---
+
+    /// When the attached vector index was last refreshed. `None` means this
+    /// dataset has no vector index yet, or has never been embedded.
+    #[serde(default)]
+    pub last_embedded_at: Option<chrono::DateTime<chrono::Utc>>,
+
+    /// When data was written that hasn't yet been embedded. `Some(t)` means
+    /// the vector index is out-of-date as of timestamp `t`. Cleared on refresh.
+    #[serde(default)]
+    pub embedding_stale_since: Option<chrono::DateTime<chrono::Utc>>,
+
+    /// How this dataset wants stale embeddings handled.
+    #[serde(default)]
+    pub embedding_refresh_policy: Option<RefreshPolicy>,
+}
+
+/// Controls what happens when new data is written to a dataset with an
+/// attached vector index.
+///
+/// - `Manual` (default): data writes set `embedding_stale_since`; nothing
+///   embeds until an operator or agent calls `/vectors/refresh/{dataset}`.
+/// - `OnAppend`: ingest fires a background refresh immediately after writing.
+///   Suitable for datasets where vector freshness matters more than ingest
+///   latency.
+/// - `Scheduled(cron)`: a timer or external scheduler triggers refresh at
+///   the named cadence. The scheduler itself is not in this ADR scope —
+///   this just declares the intent so operators can see which policy a
+///   dataset expects.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum RefreshPolicy {
+    Manual,
+    OnAppend,
+    Scheduled { cron: String },
+}
+
+impl Default for RefreshPolicy {
+    fn default() -> Self { Self::Manual }
+}
+
+/// AI-safe view of a dataset (Phase D).
+///
+/// A view is a named projection over a base dataset that:
+/// - whitelists the columns visible (everything else is invisible),
+/// - optionally applies a row filter (a SQL WHERE clause fragment),
+/// - optionally redacts column values (e.g. mask the last 4 of an SSN).
+///
+/// Queries that reference the view name see only the safe projection.
+/// The base table remains separately accessible by its raw name — this
+/// is a *capability surface*, not a data classification fence. Use
+/// access control (Phase 13) on top to enforce who can query which view.
+///
+/// Stored as JSON at `_catalog/views/{name}.json` in the primary bucket
+/// alongside dataset manifests. The view itself doesn't store data —
+/// queryd materializes it as a DataFusion view at SessionContext build
+/// time.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AiView {
+    pub name: String,
+    /// Source dataset name. Must exist in the catalog.
+    pub base_dataset: String,
+    /// Whitelisted columns. Empty list is invalid (would expose nothing).
+    pub columns: Vec<String>,
+    /// Optional WHERE clause fragment (without the `WHERE` keyword), e.g.
+    /// `status != 'blocked' AND city = 'Chicago'`. SQL injection here is
+    /// the operator's problem — only trust this from authenticated admins.
+    #[serde(default)]
+    pub row_filter: Option<String>,
+    /// Per-column redactions: `{column_name -> Redaction}`.
+    #[serde(default)]
+    pub column_redactions: std::collections::HashMap<String, Redaction>,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+    #[serde(default)]
+    pub description: String,
+}
+
+/// Per-model configuration for HNSW builds. Declared inline so the PRD
+/// invariant 9 ("every reader gets its own profile") can carry tuning
+/// preferences without forcing a round-trip to vectord's trial system.
+/// Mirrors `vectord::trial::HnswConfig` to avoid a cross-crate dep cycle.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct ProfileHnswConfig {
+    pub ef_construction: usize,
+    pub ef_search: usize,
+    #[serde(default)]
+    pub seed: Option<u64>,
+}
+
+impl Default for ProfileHnswConfig {
+    /// Matches the production HNSW default from Phase 15 trial sweep
+    /// (see docs/ADR and vectord::trial::HnswConfig::default).
+    fn default() -> Self {
+        Self { ef_construction: 80, ef_search: 30, seed: None }
+    }
+}
+
+/// A named client of the substrate — a local LLM, an AI agent, or a
+/// human role that should see a specific scoped view of the data.
+/// Profiles are the first-class "every reader gets its own profile"
+/// concept from PRD invariant 9.
+///
+/// Bound datasets can be raw catalog tables OR `AiView` names — either
+/// works at query time since both register as DataFusion tables with
+/// the same name. Mixing raw and views lets you compose access rules:
+/// bind a model to `candidates_safe` (PII-redacted view) and `placements`
+/// (raw table, no PII), and the model gets exactly what it needs.
+///
+/// Stored at `_catalog/profiles/{id}.json`. Activation (`/profile/{id}/
+/// activate`) pre-loads the embedding cache and builds HNSW with the
+/// profile's config — first search after activate is warm.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ModelProfile {
+    /// Unique id — slug form (alphanumeric + `-`, `_`). Used in URLs,
+    /// bucket names (`profile:{id}`), and trial journal prefixes.
+    pub id: String,
+    /// Ollama model tag (e.g. "qwen2.5:7b", "nomic-embed-text:latest").
+    /// Used when this profile is the "generate" or "embed" caller.
+    pub ollama_name: String,
+    /// Free-text description surfaced to operators. Who/what is this
+    /// profile for?
+    #[serde(default)]
+    pub description: String,
+    /// Names of datasets or AiViews this profile can read. A search
+    /// against an index whose `source` isn't in this list is refused.
+    pub bound_datasets: Vec<String>,
+    /// Per-profile HNSW config, used when activate builds the graph.
+    #[serde(default)]
+    pub hnsw_config: ProfileHnswConfig,
+    /// Which embedding model this profile expects. Defaults to
+    /// `nomic-embed-text` (the only one currently installed).
+    #[serde(default = "default_embed_model")]
+    pub embed_model: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(default)]
+    pub created_by: String,
+    /// Federation layer 2: which bucket this profile's artifacts
+    /// (trial journals, promotions for its indexes) live in. If set and
+    /// not yet registered, `POST /vectors/profile/{id}/activate` auto-
+    /// provisions a local bucket under `storage.profile_root` and
+    /// registers it before warming indexes. `None` = share the primary
+    /// bucket (pre-federation behavior).
+    #[serde(default)]
+    pub bucket: Option<String>,
+    /// ADR-019 hybrid: which storage backend serves this profile's
+    /// indexes. Default Parquet so pre-existing profiles unchanged.
+    /// New profiles can opt into Lance for append-heavy / random-access
+    /// / 5M+ vector workloads.
+    #[serde(default)]
+    pub vector_backend: VectorBackend,
+    /// Phase 41: Profile type for routing + activation behavior.
+    /// Default Execution for backward compatibility.
+    #[serde(default)]
+    pub profile_type: ProfileType,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Default)]
+#[serde(rename_all = "snake_case")]
+pub enum ProfileType {
+    #[default]
+    Execution,
+    Retrieval,
+    Memory,
+    Observer,
+}
+
+fn default_embed_model() -> String { "nomic-embed-text".to_string() }
+
+/// Per-profile / per-index storage backend choice.
+///
+/// `Parquet` (default) — vectors as binary-blob in Parquet + in-RAM HNSW
+/// for ANN. Best when:
+/// - Index ≤ ~5M vectors (fits in RAM)
+/// - Search latency dominates (HNSW p50 < 1ms at 100K)
+/// - Workload is mostly read, not append
+///
+/// `Lance` — vectors as native FixedSizeList in a Lance dataset, IVF_PQ
+/// on-disk index. Best when:
+/// - Index will exceed RAM ceiling (5M+ vectors per index)
+/// - Heavy random-row access for RAG (Lance: 311us; Parquet: ~35ms full
+///   scan per fetch)
+/// - Append-heavy ingest (Lance: 0.08s for 10K rows; Parquet: full
+///   ~330MB rewrite per change)
+/// - Need to rebuild index frequently (Lance IVF_PQ: 16s; HNSW: 230s
+///   at the same scale)
+///
+/// Defaults to Parquet for backward compat with pre-ADR-019 indexes.
+/// See `docs/ADR-019-vector-storage.md` for the full scorecard.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum VectorBackend {
+    Parquet,
+    Lance,
+}
+
+impl Default for VectorBackend {
+    fn default() -> Self { VectorBackend::Parquet }
+}
+
+/// Soft-delete marker (Phase E).
+///
+/// Tombstones live beside the dataset in `_catalog/tombstones/{dataset}/`
+/// as append-log JSONL. Query-time filter in queryd reads all tombstones
+/// for each dataset and wraps the base table in a DataFusion view that
+/// excludes tombstoned rows. Physical deletion happens later (compaction),
+/// so the row count immediately reflects the delete but data is still on
+/// disk until compact runs. That's deliberate — it gives a reversal
+/// window and keeps the event journal audit trail intact.
+///
+/// All tombstones for a given dataset must use the same `row_key_column`
+/// (enforced on write); otherwise the query-time filter can't be built
+/// as a single WHERE clause.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Tombstone {
+    pub dataset: String,
+    /// Column name that identifies the row (e.g. "candidate_id").
+    pub row_key_column: String,
+    /// Value of that column for the tombstoned row.
+    pub row_key_value: String,
+    pub deleted_at: chrono::DateTime<chrono::Utc>,
+    /// Human / system actor responsible for the delete (audit).
+    #[serde(default)]
+    pub actor: String,
+    /// Why (e.g. "GDPR request #1234", "user-requested erasure").
+    #[serde(default)]
+    pub reason: String,
+}
+
+/// How a column's values should be transformed before being returned.
+/// `Mask` is the most common — keeps a few visible chars, replaces the
+/// rest with `*`. `Hash` returns SHA-256 of the value for join keys you
+/// want to preserve referential integrity on without exposing the value.
+/// `Null` returns NULL unconditionally.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum Redaction {
+    Null,
+    Hash,
+    Mask {
+        /// Number of leading characters left visible.
+        keep_prefix: usize,
+        /// Number of trailing characters left visible.
+        keep_suffix: usize,
+    },
+}