Add Phase 10-12 implementation: multi-tenant, marketplace, observability
Major additions: - marketplace/: Agent template registry with FTS5 search, ratings, versioning - observability/: Prometheus metrics, distributed tracing, structured logging - ledger/migrations/: Database migration scripts for multi-tenant support - tests/governance/: 15 new test files for phases 6-12 (295 total tests) - bin/validate-phases: Full 12-phase validation script New features: - Multi-tenant support with tenant isolation and quota enforcement - Agent marketplace with semantic versioning and search - Observability with metrics, tracing, and log correlation - Tier-1 agent bootstrap scripts Updated components: - ledger/api.py: Extended API for tenants, marketplace, observability - ledger/schema.sql: Added tenant, project, marketplace tables - testing/framework.ts: Enhanced test framework - checkpoint/checkpoint.py: Improved checkpoint management Archived: - External integrations (Slack/GitHub/PagerDuty) moved to .archive/ - Old checkpoint files cleaned up Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
4b5b3b0a2d
commit
8c6e7831e9
+2
-2
@@ -2,7 +2,7 @@
|
||||
|
||||
## Current Phase
|
||||
|
||||
** NOT STARTED**
|
||||
** COMPLETE**
|
||||
|
||||
## Tasks
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
## Activity Log
|
||||
|
||||
### 2026-01-23 23:25:09 UTC
|
||||
- **Phase**: NOT STARTED
|
||||
- **Phase**: COMPLETE
|
||||
- **Action**: Initialized
|
||||
- **Details**: Status tracking initialized for this directory.
|
||||
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tenant-Aware Redis Client
|
||||
=========================
|
||||
Provides tenant-prefixed Redis operations for multi-tenant data isolation.
|
||||
|
||||
Usage:
|
||||
from lib.tenant_redis import TenantRedis
|
||||
|
||||
redis = TenantRedis(tenant_id="acme", project_id="web-app")
|
||||
redis.set("checkpoint:latest", data) # Stored as: tenant:acme:project:web-app:checkpoint:latest
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Optional, Any, List
|
||||
import redis
|
||||
|
||||
|
||||
class TenantRedis:
|
||||
"""Redis client with automatic tenant/project key prefixing"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tenant_id: str = "default",
|
||||
project_id: str = "default",
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 6379,
|
||||
password: Optional[str] = None,
|
||||
db: int = 0
|
||||
):
|
||||
self.tenant_id = tenant_id
|
||||
self.project_id = project_id
|
||||
|
||||
# Get password from environment or Vault
|
||||
if password is None:
|
||||
password = os.environ.get("DRAGONFLY_PASSWORD", "governance2026")
|
||||
|
||||
self._client = redis.Redis(
|
||||
host=host,
|
||||
port=port,
|
||||
password=password,
|
||||
db=db,
|
||||
decode_responses=True
|
||||
)
|
||||
|
||||
def _prefix(self, key: str) -> str:
|
||||
"""Add tenant/project prefix to key"""
|
||||
return f"tenant:{self.tenant_id}:project:{self.project_id}:{key}"
|
||||
|
||||
def _global_prefix(self, key: str) -> str:
|
||||
"""Add tenant-only prefix for cross-project data"""
|
||||
return f"tenant:{self.tenant_id}:{key}"
|
||||
|
||||
# =========================================================================
|
||||
# Basic Operations (Project-Scoped)
|
||||
# =========================================================================
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
"""Get value by key (project-scoped)"""
|
||||
return self._client.get(self._prefix(key))
|
||||
|
||||
def set(self, key: str, value: Any, ex: Optional[int] = None, nx: bool = False) -> bool:
|
||||
"""Set value by key (project-scoped)"""
|
||||
if isinstance(value, (dict, list)):
|
||||
value = json.dumps(value)
|
||||
return self._client.set(self._prefix(key), value, ex=ex, nx=nx)
|
||||
|
||||
def delete(self, key: str) -> int:
|
||||
"""Delete key (project-scoped)"""
|
||||
return self._client.delete(self._prefix(key))
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if key exists (project-scoped)"""
|
||||
return self._client.exists(self._prefix(key)) > 0
|
||||
|
||||
def keys(self, pattern: str) -> List[str]:
|
||||
"""Find keys matching pattern (project-scoped)"""
|
||||
full_pattern = self._prefix(pattern)
|
||||
keys = self._client.keys(full_pattern)
|
||||
# Strip prefix from results
|
||||
prefix_len = len(self._prefix(""))
|
||||
return [k[prefix_len:] for k in keys]
|
||||
|
||||
def expire(self, key: str, seconds: int) -> bool:
|
||||
"""Set key expiration (project-scoped)"""
|
||||
return self._client.expire(self._prefix(key), seconds)
|
||||
|
||||
def ttl(self, key: str) -> int:
|
||||
"""Get key TTL (project-scoped)"""
|
||||
return self._client.ttl(self._prefix(key))
|
||||
|
||||
# =========================================================================
|
||||
# Hash Operations (Project-Scoped)
|
||||
# =========================================================================
|
||||
|
||||
def hget(self, key: str, field: str) -> Optional[str]:
|
||||
"""Get hash field"""
|
||||
return self._client.hget(self._prefix(key), field)
|
||||
|
||||
def hset(self, key: str, field: str, value: Any) -> int:
|
||||
"""Set hash field"""
|
||||
if isinstance(value, (dict, list)):
|
||||
value = json.dumps(value)
|
||||
return self._client.hset(self._prefix(key), field, value)
|
||||
|
||||
def hgetall(self, key: str) -> dict:
|
||||
"""Get all hash fields"""
|
||||
return self._client.hgetall(self._prefix(key))
|
||||
|
||||
def hincrby(self, key: str, field: str, amount: int = 1) -> int:
|
||||
"""Increment hash field"""
|
||||
return self._client.hincrby(self._prefix(key), field, amount)
|
||||
|
||||
# =========================================================================
|
||||
# List Operations (Project-Scoped)
|
||||
# =========================================================================
|
||||
|
||||
def lpush(self, key: str, *values) -> int:
|
||||
"""Push to list head"""
|
||||
serialized = [json.dumps(v) if isinstance(v, (dict, list)) else v for v in values]
|
||||
return self._client.lpush(self._prefix(key), *serialized)
|
||||
|
||||
def rpush(self, key: str, *values) -> int:
|
||||
"""Push to list tail"""
|
||||
serialized = [json.dumps(v) if isinstance(v, (dict, list)) else v for v in values]
|
||||
return self._client.rpush(self._prefix(key), *serialized)
|
||||
|
||||
def lpop(self, key: str) -> Optional[str]:
|
||||
"""Pop from list head"""
|
||||
return self._client.lpop(self._prefix(key))
|
||||
|
||||
def rpop(self, key: str) -> Optional[str]:
|
||||
"""Pop from list tail"""
|
||||
return self._client.rpop(self._prefix(key))
|
||||
|
||||
def lrange(self, key: str, start: int, end: int) -> List[str]:
|
||||
"""Get list range"""
|
||||
return self._client.lrange(self._prefix(key), start, end)
|
||||
|
||||
def llen(self, key: str) -> int:
|
||||
"""Get list length"""
|
||||
return self._client.llen(self._prefix(key))
|
||||
|
||||
def ltrim(self, key: str, start: int, end: int) -> bool:
|
||||
"""Trim list to range"""
|
||||
return self._client.ltrim(self._prefix(key), start, end)
|
||||
|
||||
# =========================================================================
|
||||
# Set Operations (Project-Scoped)
|
||||
# =========================================================================
|
||||
|
||||
def sadd(self, key: str, *members) -> int:
|
||||
"""Add to set"""
|
||||
return self._client.sadd(self._prefix(key), *members)
|
||||
|
||||
def srem(self, key: str, *members) -> int:
|
||||
"""Remove from set"""
|
||||
return self._client.srem(self._prefix(key), *members)
|
||||
|
||||
def smembers(self, key: str) -> set:
|
||||
"""Get set members"""
|
||||
return self._client.smembers(self._prefix(key))
|
||||
|
||||
def sismember(self, key: str, member: str) -> bool:
|
||||
"""Check set membership"""
|
||||
return self._client.sismember(self._prefix(key), member)
|
||||
|
||||
# =========================================================================
|
||||
# Tenant-Level Operations (Cross-Project)
|
||||
# =========================================================================
|
||||
|
||||
def tenant_get(self, key: str) -> Optional[str]:
|
||||
"""Get value by key (tenant-level, cross-project)"""
|
||||
return self._client.get(self._global_prefix(key))
|
||||
|
||||
def tenant_set(self, key: str, value: Any, ex: Optional[int] = None) -> bool:
|
||||
"""Set value by key (tenant-level, cross-project)"""
|
||||
if isinstance(value, (dict, list)):
|
||||
value = json.dumps(value)
|
||||
return self._client.set(self._global_prefix(key), value, ex=ex)
|
||||
|
||||
def tenant_lpush(self, key: str, *values) -> int:
|
||||
"""Push to list (tenant-level)"""
|
||||
serialized = [json.dumps(v) if isinstance(v, (dict, list)) else v for v in values]
|
||||
return self._client.lpush(self._global_prefix(key), *serialized)
|
||||
|
||||
def tenant_lrange(self, key: str, start: int, end: int) -> List[str]:
|
||||
"""Get list range (tenant-level)"""
|
||||
return self._client.lrange(self._global_prefix(key), start, end)
|
||||
|
||||
# =========================================================================
|
||||
# Global Operations (No Prefix - Admin Only)
|
||||
# =========================================================================
|
||||
|
||||
def global_get(self, key: str) -> Optional[str]:
|
||||
"""Get value without prefix (admin use)"""
|
||||
return self._client.get(key)
|
||||
|
||||
def global_set(self, key: str, value: Any, ex: Optional[int] = None) -> bool:
|
||||
"""Set value without prefix (admin use)"""
|
||||
if isinstance(value, (dict, list)):
|
||||
value = json.dumps(value)
|
||||
return self._client.set(key, value, ex=ex)
|
||||
|
||||
def global_keys(self, pattern: str) -> List[str]:
|
||||
"""Find keys without prefix (admin use)"""
|
||||
return self._client.keys(pattern)
|
||||
|
||||
# =========================================================================
|
||||
# Utility
|
||||
# =========================================================================
|
||||
|
||||
def ping(self) -> bool:
|
||||
"""Check connection"""
|
||||
try:
|
||||
return self._client.ping()
|
||||
except:
|
||||
return False
|
||||
|
||||
def info(self) -> dict:
|
||||
"""Get Redis info"""
|
||||
return self._client.info()
|
||||
|
||||
def switch_context(self, tenant_id: str = None, project_id: str = None) -> 'TenantRedis':
|
||||
"""Create new client with different tenant/project context"""
|
||||
return TenantRedis(
|
||||
tenant_id=tenant_id or self.tenant_id,
|
||||
project_id=project_id or self.project_id,
|
||||
host=self._client.connection_pool.connection_kwargs.get('host', '127.0.0.1'),
|
||||
port=self._client.connection_pool.connection_kwargs.get('port', 6379),
|
||||
password=self._client.connection_pool.connection_kwargs.get('password'),
|
||||
db=self._client.connection_pool.connection_kwargs.get('db', 0)
|
||||
)
|
||||
|
||||
@property
|
||||
def prefix(self) -> str:
|
||||
"""Get current key prefix"""
|
||||
return self._prefix("")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Factory function for easy creation
|
||||
# =============================================================================
|
||||
|
||||
def get_tenant_redis(
|
||||
tenant_id: str = "default",
|
||||
project_id: str = "default"
|
||||
) -> TenantRedis:
|
||||
"""
|
||||
Create a TenantRedis client with given context.
|
||||
|
||||
Environment variables:
|
||||
DRAGONFLY_HOST: Redis host (default: 127.0.0.1)
|
||||
DRAGONFLY_PORT: Redis port (default: 6379)
|
||||
DRAGONFLY_PASSWORD: Redis password
|
||||
"""
|
||||
return TenantRedis(
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
host=os.environ.get("DRAGONFLY_HOST", "127.0.0.1"),
|
||||
port=int(os.environ.get("DRAGONFLY_PORT", "6379")),
|
||||
password=os.environ.get("DRAGONFLY_PASSWORD", "governance2026")
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Migration helper - for transitioning existing keys
|
||||
# =============================================================================
|
||||
|
||||
def migrate_keys_to_tenant(
|
||||
source_redis: redis.Redis,
|
||||
tenant_id: str = "default",
|
||||
project_id: str = "default",
|
||||
patterns: List[str] = None,
|
||||
dry_run: bool = True
|
||||
) -> dict:
|
||||
"""
|
||||
Migrate existing non-prefixed keys to tenant-prefixed format.
|
||||
|
||||
Args:
|
||||
source_redis: Existing Redis connection
|
||||
tenant_id: Target tenant ID
|
||||
project_id: Target project ID
|
||||
patterns: Key patterns to migrate (default: common governance keys)
|
||||
dry_run: If True, only report what would be migrated
|
||||
|
||||
Returns:
|
||||
dict with migration results
|
||||
"""
|
||||
if patterns is None:
|
||||
patterns = [
|
||||
"checkpoint:*",
|
||||
"agent:*",
|
||||
"task:*",
|
||||
"orchestration:*",
|
||||
"revocations:*",
|
||||
"alerts:*"
|
||||
]
|
||||
|
||||
prefix = f"tenant:{tenant_id}:project:{project_id}:"
|
||||
results = {
|
||||
"migrated": [],
|
||||
"skipped": [],
|
||||
"errors": []
|
||||
}
|
||||
|
||||
for pattern in patterns:
|
||||
keys = source_redis.keys(pattern)
|
||||
for key in keys:
|
||||
# Skip if already prefixed
|
||||
if key.startswith("tenant:"):
|
||||
results["skipped"].append(key)
|
||||
continue
|
||||
|
||||
new_key = f"{prefix}{key}"
|
||||
try:
|
||||
if dry_run:
|
||||
results["migrated"].append({"old": key, "new": new_key, "dry_run": True})
|
||||
else:
|
||||
# Get key type and copy appropriately
|
||||
key_type = source_redis.type(key)
|
||||
if key_type == "string":
|
||||
value = source_redis.get(key)
|
||||
source_redis.set(new_key, value)
|
||||
elif key_type == "list":
|
||||
values = source_redis.lrange(key, 0, -1)
|
||||
if values:
|
||||
source_redis.rpush(new_key, *values)
|
||||
elif key_type == "hash":
|
||||
values = source_redis.hgetall(key)
|
||||
if values:
|
||||
source_redis.hset(new_key, mapping=values)
|
||||
elif key_type == "set":
|
||||
values = source_redis.smembers(key)
|
||||
if values:
|
||||
source_redis.sadd(new_key, *values)
|
||||
|
||||
results["migrated"].append({"old": key, "new": new_key})
|
||||
except Exception as e:
|
||||
results["errors"].append({"key": key, "error": str(e)})
|
||||
|
||||
return results
|
||||
Reference in New Issue
Block a user