Phase 5: hardening — gRPC, observability, auth, config

- proto: lakehouse.proto with CatalogService, QueryService, StorageService, AiService - proto crate: tonic-build codegen from proto definitions - catalogd: gRPC CatalogService implementation - gateway: dual HTTP (:3100) + gRPC (:3101) servers - gateway: OpenTelemetry tracing with stdout exporter - gateway: API key auth middleware (toggleable) - shared: TOML config system with typed structs and defaults - lakehouse.toml config file - ADR-006 and ADR-007 documented Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 06:37:07 -05:00
parent 50a8c8013f
commit 01373c0e45
23 changed files with 888 additions and 60 deletions
@@ -0,0 +1,166 @@
+syntax = "proto3";
+package lakehouse;
+
+// --- Catalog Service ---
+
+service CatalogService {
+    rpc CreateDataset(CreateDatasetRequest) returns (DatasetResponse);
+    rpc GetDataset(GetDatasetRequest) returns (DatasetResponse);
+    rpc GetDatasetByName(GetDatasetByNameRequest) returns (DatasetResponse);
+    rpc ListDatasets(ListDatasetsRequest) returns (ListDatasetsResponse);
+}
+
+message CreateDatasetRequest {
+    string name = 1;
+    string schema_fingerprint = 2;
+    repeated ObjectRef objects = 3;
+}
+
+message GetDatasetRequest {
+    string id = 1;
+}
+
+message GetDatasetByNameRequest {
+    string name = 1;
+}
+
+message ListDatasetsRequest {}
+
+message ListDatasetsResponse {
+    repeated DatasetResponse datasets = 1;
+}
+
+message DatasetResponse {
+    string id = 1;
+    string name = 2;
+    string schema_fingerprint = 3;
+    repeated ObjectRef objects = 4;
+    string created_at = 5;
+    string updated_at = 6;
+}
+
+message ObjectRef {
+    string bucket = 1;
+    string key = 2;
+    uint64 size_bytes = 3;
+    string created_at = 4;
+}
+
+// --- Query Service ---
+
+service QueryService {
+    rpc ExecuteQuery(QueryRequest) returns (QueryResponse);
+}
+
+message QueryRequest {
+    string sql = 1;
+}
+
+message QueryResponse {
+    repeated ColumnInfo columns = 1;
+    bytes rows_json = 2;  // JSON-encoded rows
+    uint64 row_count = 3;
+}
+
+message ColumnInfo {
+    string name = 1;
+    string data_type = 2;
+}
+
+// --- Storage Service ---
+
+service StorageService {
+    rpc PutObject(PutObjectRequest) returns (PutObjectResponse);
+    rpc GetObject(GetObjectRequest) returns (GetObjectResponse);
+    rpc DeleteObject(DeleteObjectRequest) returns (DeleteObjectResponse);
+    rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse);
+}
+
+message PutObjectRequest {
+    string key = 1;
+    bytes data = 2;
+}
+
+message PutObjectResponse {
+    string key = 1;
+}
+
+message GetObjectRequest {
+    string key = 1;
+}
+
+message GetObjectResponse {
+    bytes data = 1;
+}
+
+message DeleteObjectRequest {
+    string key = 1;
+}
+
+message DeleteObjectResponse {
+    string key = 1;
+}
+
+message ListObjectsRequest {
+    string prefix = 1;
+}
+
+message ListObjectsResponse {
+    repeated string keys = 1;
+}
+
+// --- AI Service ---
+
+service AiService {
+    rpc Embed(EmbedRequest) returns (EmbedResponse);
+    rpc Generate(GenerateRequest) returns (GenerateResponse);
+    rpc Rerank(RerankRequest) returns (RerankResponse);
+}
+
+message EmbedRequest {
+    repeated string texts = 1;
+    string model = 2;
+}
+
+message EmbedResponse {
+    repeated Embedding embeddings = 1;
+    string model = 2;
+    uint32 dimensions = 3;
+}
+
+message Embedding {
+    repeated double values = 1;
+}
+
+message GenerateRequest {
+    string prompt = 1;
+    string model = 2;
+    string system = 3;
+    double temperature = 4;
+    uint32 max_tokens = 5;
+}
+
+message GenerateResponse {
+    string text = 1;
+    string model = 2;
+    uint64 tokens_evaluated = 3;
+    uint64 tokens_generated = 4;
+}
+
+message RerankRequest {
+    string query = 1;
+    repeated string documents = 2;
+    string model = 3;
+    uint32 top_k = 4;
+}
+
+message RerankResponse {
+    repeated ScoredDocument results = 1;
+    string model = 2;
+}
+
+message ScoredDocument {
+    uint32 index = 1;
+    string text = 2;
+    double score = 3;
+}