Phase 5: hardening — gRPC, observability, auth, config

- proto: lakehouse.proto with CatalogService, QueryService, StorageService, AiService
- proto crate: tonic-build codegen from proto definitions
- catalogd: gRPC CatalogService implementation
- gateway: dual HTTP (:3100) + gRPC (:3101) servers
- gateway: OpenTelemetry tracing with stdout exporter
- gateway: API key auth middleware (toggleable)
- shared: TOML config system with typed structs and defaults
- lakehouse.toml config file
- ADR-006 and ADR-007 documented

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/shared/Cargo.toml

@@ -13,3 +13,5 @@ arrow = { workspace = true }
 parquet = { workspace = true }
 bytes = { workspace = true }
 sha2 = { workspace = true }
+toml = { workspace = true }
+tracing = { workspace = true }

crates/shared/src/config.rs

@@ -0,0 +1,127 @@
+use serde::Deserialize;
+use std::path::Path;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Config {
+    pub gateway: GatewayConfig,
+    pub storage: StorageConfig,
+    #[serde(default)]
+    pub catalog: CatalogConfig,
+    #[serde(default)]
+    pub query: QueryConfig,
+    pub sidecar: SidecarConfig,
+    #[serde(default)]
+    pub ai: AiConfig,
+    #[serde(default)]
+    pub auth: AuthConfig,
+    #[serde(default)]
+    pub observability: ObservabilityConfig,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct GatewayConfig {
+    #[serde(default = "default_host")]
+    pub host: String,
+    #[serde(default = "default_gateway_port")]
+    pub port: u16,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct StorageConfig {
+    #[serde(default = "default_storage_root")]
+    pub root: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct CatalogConfig {
+    #[serde(default = "default_manifest_prefix")]
+    pub manifest_prefix: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct QueryConfig {
+    pub max_rows_per_query: Option<usize>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct SidecarConfig {
+    #[serde(default = "default_sidecar_url")]
+    pub url: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct AiConfig {
+    #[serde(default = "default_embed_model")]
+    pub embed_model: String,
+    #[serde(default = "default_gen_model")]
+    pub gen_model: String,
+    #[serde(default = "default_rerank_model")]
+    pub rerank_model: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct AuthConfig {
+    #[serde(default)]
+    pub enabled: bool,
+    pub api_key: Option<String>,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct ObservabilityConfig {
+    #[serde(default = "default_exporter")]
+    pub exporter: String,
+    #[serde(default = "default_service_name")]
+    pub service_name: String,
+}
+
+// Defaults
+fn default_host() -> String { "0.0.0.0".to_string() }
+fn default_gateway_port() -> u16 { 3100 }
+fn default_storage_root() -> String { "./data".to_string() }
+fn default_manifest_prefix() -> String { "_catalog/manifests".to_string() }
+fn default_sidecar_url() -> String { "http://localhost:3200".to_string() }
+fn default_embed_model() -> String { "nomic-embed-text".to_string() }
+fn default_gen_model() -> String { "qwen2.5".to_string() }
+fn default_rerank_model() -> String { "qwen2.5".to_string() }
+fn default_exporter() -> String { "stdout".to_string() }
+fn default_service_name() -> String { "lakehouse".to_string() }
+
+impl Config {
+    pub fn load(path: &str) -> Result<Self, String> {
+        let path = Path::new(path);
+        if !path.exists() {
+            return Err(format!("config file not found: {}", path.display()));
+        }
+        let content = std::fs::read_to_string(path)
+            .map_err(|e| format!("failed to read config: {e}"))?;
+        toml::from_str(&content)
+            .map_err(|e| format!("failed to parse config: {e}"))
+    }
+
+    pub fn load_or_default() -> Self {
+        // Try lakehouse.toml in current dir, then /etc/lakehouse/lakehouse.toml
+        for path in &["lakehouse.toml", "/etc/lakehouse/lakehouse.toml"] {
+            if let Ok(config) = Self::load(path) {
+                tracing::info!("loaded config from {path}");
+                return config;
+            }
+        }
+        tracing::warn!("no config file found, using defaults");
+        Self::default()
+    }
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            gateway: GatewayConfig { host: default_host(), port: default_gateway_port() },
+            storage: StorageConfig { root: default_storage_root() },
+            catalog: CatalogConfig::default(),
+            query: QueryConfig::default(),
+            sidecar: SidecarConfig { url: default_sidecar_url() },
+            ai: AiConfig::default(),
+            auth: AuthConfig::default(),
+            observability: ObservabilityConfig::default(),
+        }
+    }
+}

crates/shared/src/lib.rs

@@ -1,3 +1,4 @@
 pub mod types;
 pub mod errors;
 pub mod arrow_helpers;
+pub mod config;