Add Phase 10-12 implementation: multi-tenant, marketplace, observability
Major additions: - marketplace/: Agent template registry with FTS5 search, ratings, versioning - observability/: Prometheus metrics, distributed tracing, structured logging - ledger/migrations/: Database migration scripts for multi-tenant support - tests/governance/: 15 new test files for phases 6-12 (295 total tests) - bin/validate-phases: Full 12-phase validation script New features: - Multi-tenant support with tenant isolation and quota enforcement - Agent marketplace with semantic versioning and search - Observability with metrics, tracing, and log correlation - Tier-1 agent bootstrap scripts Updated components: - ledger/api.py: Extended API for tenants, marketplace, observability - ledger/schema.sql: Added tenant, project, marketplace tables - testing/framework.ts: Enhanced test framework - checkpoint/checkpoint.py: Improved checkpoint management Archived: - External integrations (Slack/GitHub/PagerDuty) moved to .archive/ - Old checkpoint files cleaned up Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
4b5b3b0a2d
commit
8c6e7831e9
@@ -16,7 +16,7 @@ from datetime import datetime, timezone
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Optional, cast
|
||||
import redis
|
||||
|
||||
from .bug_watcher import BugWindowWatcher, Anomaly
|
||||
@@ -24,6 +24,7 @@ from .bug_watcher import BugWindowWatcher, Anomaly
|
||||
|
||||
class PhaseStatus(str, Enum):
|
||||
"""Status of a phase"""
|
||||
|
||||
NOT_STARTED = "not_started"
|
||||
IN_PROGRESS = "in_progress"
|
||||
COMPLETE = "complete"
|
||||
@@ -33,8 +34,9 @@ class PhaseStatus(str, Enum):
|
||||
|
||||
class ValidationLevel(str, Enum):
|
||||
"""Level of validation applied"""
|
||||
|
||||
NONE = "none"
|
||||
BASIC = "basic" # Existence checks only
|
||||
BASIC = "basic" # Existence checks only
|
||||
STANDARD = "standard" # + functionality tests
|
||||
THOROUGH = "thorough" # + integration tests
|
||||
COMPREHENSIVE = "comprehensive" # + chaos/edge cases
|
||||
@@ -43,6 +45,7 @@ class ValidationLevel(str, Enum):
|
||||
@dataclass
|
||||
class PhaseDefinition:
|
||||
"""Definition of a phase"""
|
||||
|
||||
number: int
|
||||
name: str
|
||||
description: str
|
||||
@@ -57,6 +60,7 @@ class PhaseDefinition:
|
||||
@dataclass
|
||||
class PhaseValidationResult:
|
||||
"""Result of validating a phase"""
|
||||
|
||||
phase_number: int
|
||||
phase_name: str
|
||||
status: PhaseStatus
|
||||
@@ -91,36 +95,30 @@ class PhaseValidator:
|
||||
name="Foundation (Vault + Basic Infrastructure)",
|
||||
description="Vault installation, TLS, audit logging, ledger setup",
|
||||
key_directories=["ledger", "bin"],
|
||||
key_files=[
|
||||
"ledger/governance.db",
|
||||
"ledger/schema.sql",
|
||||
"ledger/api.py"
|
||||
],
|
||||
key_files=["ledger/governance.db", "ledger/schema.sql", "ledger/api.py"],
|
||||
required_tests=["ledger_connection", "vault_status", "audit_logging"],
|
||||
dependencies=[],
|
||||
validation_criteria=[
|
||||
"Vault accessible and unsealed",
|
||||
"Ledger database operational",
|
||||
"Audit logging enabled"
|
||||
"Audit logging enabled",
|
||||
],
|
||||
priority="critical"
|
||||
priority="critical",
|
||||
),
|
||||
2: PhaseDefinition(
|
||||
number=2,
|
||||
name="Vault Policy Engine",
|
||||
description="Trust tier policies, secrets engines, AppRole auth",
|
||||
key_directories=["runtime"],
|
||||
key_files=[
|
||||
"runtime/governance.py"
|
||||
],
|
||||
key_files=["runtime/governance.py"],
|
||||
required_tests=["policy_enforcement", "secrets_access", "approle_auth"],
|
||||
dependencies=[1],
|
||||
validation_criteria=[
|
||||
"All tier policies loaded",
|
||||
"SSH and KV engines configured",
|
||||
"AppRole roles created"
|
||||
"AppRole roles created",
|
||||
],
|
||||
priority="critical"
|
||||
priority="critical",
|
||||
),
|
||||
3: PhaseDefinition(
|
||||
number=3,
|
||||
@@ -131,16 +129,20 @@ class PhaseValidator:
|
||||
"preflight/preflight.py",
|
||||
"wrappers/tf-governed.sh",
|
||||
"wrappers/ansible-governed.sh",
|
||||
"evidence/evidence.py"
|
||||
"evidence/evidence.py",
|
||||
],
|
||||
required_tests=[
|
||||
"preflight_gate",
|
||||
"wrapper_enforcement",
|
||||
"evidence_collection",
|
||||
],
|
||||
required_tests=["preflight_gate", "wrapper_enforcement", "evidence_collection"],
|
||||
dependencies=[1, 2],
|
||||
validation_criteria=[
|
||||
"Preflight blocks unauthorized targets",
|
||||
"Wrappers enforce plan-first",
|
||||
"Evidence packages generated"
|
||||
"Evidence packages generated",
|
||||
],
|
||||
priority="critical"
|
||||
priority="critical",
|
||||
),
|
||||
4: PhaseDefinition(
|
||||
number=4,
|
||||
@@ -150,16 +152,16 @@ class PhaseValidator:
|
||||
key_files=[
|
||||
"runtime/promotion.py",
|
||||
"runtime/revocation.py",
|
||||
"runtime/monitors.py"
|
||||
"runtime/monitors.py",
|
||||
],
|
||||
required_tests=["promotion_logic", "revocation_triggers", "monitor_daemon"],
|
||||
dependencies=[1, 2, 3],
|
||||
validation_criteria=[
|
||||
"Promotion requirements enforced",
|
||||
"Violations trigger revocation",
|
||||
"Monitors detect issues"
|
||||
"Monitors detect issues",
|
||||
],
|
||||
priority="critical"
|
||||
priority="critical",
|
||||
),
|
||||
5: PhaseDefinition(
|
||||
number=5,
|
||||
@@ -169,57 +171,55 @@ class PhaseValidator:
|
||||
key_files=[
|
||||
"checkpoint/checkpoint.py",
|
||||
"agents/tier0-agent/agent.py",
|
||||
"orchestrator/model_controller.py"
|
||||
"orchestrator/model_controller.py",
|
||||
],
|
||||
required_tests=[
|
||||
"checkpoint_create_load",
|
||||
"tier0_agent_constraints",
|
||||
"orchestrator_delegation",
|
||||
"context_preservation"
|
||||
"context_preservation",
|
||||
],
|
||||
dependencies=[1, 2, 3, 4],
|
||||
validation_criteria=[
|
||||
"Checkpoints preserve context",
|
||||
"Tier 0 agent read-only",
|
||||
"Orchestrator delegates safely"
|
||||
"Orchestrator delegates safely",
|
||||
],
|
||||
priority="critical" # SPECIAL ATTENTION
|
||||
priority="critical", # SPECIAL ATTENTION
|
||||
),
|
||||
6: PhaseDefinition(
|
||||
number=6,
|
||||
name="Pipeline DSL, Agent Templates, Testing Framework",
|
||||
description="Pipeline definitions, agent templates, test suites",
|
||||
key_directories=["pipeline", "tests"],
|
||||
key_files=[
|
||||
"pipeline/core.py",
|
||||
"pipeline/pipeline.py"
|
||||
key_files=["pipeline/core.py", "pipeline/pipeline.py"],
|
||||
required_tests=[
|
||||
"pipeline_validation",
|
||||
"template_generation",
|
||||
"test_execution",
|
||||
],
|
||||
required_tests=["pipeline_validation", "template_generation", "test_execution"],
|
||||
dependencies=[1, 2, 3, 4, 5],
|
||||
validation_criteria=[
|
||||
"Pipeline DSL parses correctly",
|
||||
"Templates generate valid agents",
|
||||
"Test suites pass"
|
||||
"Test suites pass",
|
||||
],
|
||||
priority="high"
|
||||
priority="high",
|
||||
),
|
||||
7: PhaseDefinition(
|
||||
number=7,
|
||||
name="Hierarchical Teams & Learning System",
|
||||
description="Team framework, analytics, memory layer",
|
||||
key_directories=["teams", "analytics", "memory"],
|
||||
key_files=[
|
||||
"teams/framework/team.py",
|
||||
"memory/memory.py"
|
||||
],
|
||||
key_files=["teams/framework/team.py", "memory/memory.py"],
|
||||
required_tests=["team_coordination", "learning_patterns", "memory_storage"],
|
||||
dependencies=[1, 2, 3, 4, 5, 6],
|
||||
validation_criteria=[
|
||||
"Teams delegate work",
|
||||
"Learning captures patterns",
|
||||
"Memory persists across sessions"
|
||||
"Memory persists across sessions",
|
||||
],
|
||||
priority="high"
|
||||
priority="high",
|
||||
),
|
||||
8: PhaseDefinition(
|
||||
number=8,
|
||||
@@ -229,35 +229,37 @@ class PhaseValidator:
|
||||
key_files=[
|
||||
"runtime/health_manager.py",
|
||||
"runtime/circuit_breaker.py",
|
||||
"testing/oversight/pipeline.py"
|
||||
"testing/oversight/pipeline.py",
|
||||
],
|
||||
required_tests=[
|
||||
"health_checks",
|
||||
"circuit_breaker_states",
|
||||
"alert_delivery",
|
||||
"slo_tracking",
|
||||
],
|
||||
required_tests=["health_checks", "circuit_breaker_states", "alert_delivery", "slo_tracking"],
|
||||
dependencies=[1, 2, 3, 4, 5],
|
||||
validation_criteria=[
|
||||
"Health endpoints respond",
|
||||
"Circuit breakers trip on failure",
|
||||
"Alerts delivered",
|
||||
"SLOs tracked"
|
||||
"SLOs tracked",
|
||||
],
|
||||
priority="high"
|
||||
priority="high",
|
||||
),
|
||||
9: PhaseDefinition(
|
||||
number=9,
|
||||
name="External Integrations",
|
||||
description="GitHub, Slack, webhooks",
|
||||
key_directories=["integrations"],
|
||||
key_files=[
|
||||
"integrations/github/github.py",
|
||||
"integrations/slack/slack.py"
|
||||
],
|
||||
key_files=["integrations/github/github.py", "integrations/slack/slack.py"],
|
||||
required_tests=["github_webhook", "slack_notification", "webhook_delivery"],
|
||||
dependencies=[1, 2, 3, 4, 5, 8],
|
||||
validation_criteria=[
|
||||
"GitHub integration works",
|
||||
"Slack alerts delivered",
|
||||
"Webhooks configured"
|
||||
"Webhooks configured",
|
||||
],
|
||||
priority="medium"
|
||||
priority="medium",
|
||||
),
|
||||
10: PhaseDefinition(
|
||||
number=10,
|
||||
@@ -270,9 +272,9 @@ class PhaseValidator:
|
||||
validation_criteria=[
|
||||
"Tenants isolated",
|
||||
"Quotas enforced",
|
||||
"Access controlled"
|
||||
"Access controlled",
|
||||
],
|
||||
priority="low"
|
||||
priority="low",
|
||||
),
|
||||
11: PhaseDefinition(
|
||||
number=11,
|
||||
@@ -285,26 +287,24 @@ class PhaseValidator:
|
||||
validation_criteria=[
|
||||
"Templates shareable",
|
||||
"Versions tracked",
|
||||
"Discovery works"
|
||||
"Discovery works",
|
||||
],
|
||||
priority="low"
|
||||
priority="low",
|
||||
),
|
||||
12: PhaseDefinition(
|
||||
number=12,
|
||||
name="Observability",
|
||||
description="Distributed tracing, dashboards, log aggregation",
|
||||
key_directories=["analytics", "ui"],
|
||||
key_files=[
|
||||
"ui/server.ts"
|
||||
],
|
||||
key_files=["ui/server.ts"],
|
||||
required_tests=["tracing", "dashboard_metrics", "log_aggregation"],
|
||||
dependencies=[1, 2, 3, 4, 5, 8],
|
||||
validation_criteria=[
|
||||
"Traces captured",
|
||||
"Dashboards display metrics",
|
||||
"Logs aggregated"
|
||||
"Logs aggregated",
|
||||
],
|
||||
priority="medium"
|
||||
priority="medium",
|
||||
),
|
||||
}
|
||||
|
||||
@@ -318,10 +318,10 @@ class PhaseValidator:
|
||||
"""Connect to DragonflyDB"""
|
||||
try:
|
||||
self._redis = redis.Redis(
|
||||
host='127.0.0.1',
|
||||
host="127.0.0.1",
|
||||
port=6379,
|
||||
password='governance2026',
|
||||
decode_responses=True
|
||||
password="governance2026",
|
||||
decode_responses=True,
|
||||
)
|
||||
self._redis.ping()
|
||||
except Exception:
|
||||
@@ -330,7 +330,9 @@ class PhaseValidator:
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def validate_all_phases(self, level: ValidationLevel = ValidationLevel.STANDARD) -> dict[int, PhaseValidationResult]:
|
||||
def validate_all_phases(
|
||||
self, level: ValidationLevel = ValidationLevel.STANDARD
|
||||
) -> dict[int, PhaseValidationResult]:
|
||||
"""Validate all 12 phases"""
|
||||
results = {}
|
||||
|
||||
@@ -343,7 +345,9 @@ class PhaseValidator:
|
||||
|
||||
return results
|
||||
|
||||
def validate_phase(self, phase_num: int, level: ValidationLevel = ValidationLevel.STANDARD) -> PhaseValidationResult:
|
||||
def validate_phase(
|
||||
self, phase_num: int, level: ValidationLevel = ValidationLevel.STANDARD
|
||||
) -> PhaseValidationResult:
|
||||
"""Validate a specific phase"""
|
||||
phase_def = self.PHASES.get(phase_num)
|
||||
|
||||
@@ -360,7 +364,7 @@ class PhaseValidator:
|
||||
anomalies_found=0,
|
||||
council_reviews=0,
|
||||
coverage_percent=0.0,
|
||||
gaps=["Phase not defined"]
|
||||
gaps=["Phase not defined"],
|
||||
)
|
||||
|
||||
gaps = []
|
||||
@@ -404,7 +408,11 @@ class PhaseValidator:
|
||||
council_reviews = self._get_council_review_count(phase_num)
|
||||
|
||||
# Calculate coverage
|
||||
total_items = len(phase_def.key_directories) + len(phase_def.key_files) + len(phase_def.required_tests)
|
||||
total_items = (
|
||||
len(phase_def.key_directories)
|
||||
+ len(phase_def.key_files)
|
||||
+ len(phase_def.required_tests)
|
||||
)
|
||||
found_items = dirs_found + files_found + tests_found
|
||||
coverage = (found_items / total_items * 100) if total_items > 0 else 0
|
||||
|
||||
@@ -441,7 +449,7 @@ class PhaseValidator:
|
||||
council_reviews=council_reviews,
|
||||
coverage_percent=coverage,
|
||||
gaps=gaps,
|
||||
recommendations=recommendations
|
||||
recommendations=recommendations,
|
||||
)
|
||||
|
||||
self.results[phase_num] = result
|
||||
@@ -460,6 +468,25 @@ class PhaseValidator:
|
||||
if list(self.base_path.glob(pattern)):
|
||||
return True
|
||||
|
||||
# Fall back to scanning test contents for named tests
|
||||
python_tests = list(self.base_path.glob("tests/**/*.py"))
|
||||
for test_file in python_tests:
|
||||
try:
|
||||
content = test_file.read_text()
|
||||
except OSError:
|
||||
continue
|
||||
if f"def {test_name}" in content or f"def test_{test_name}" in content:
|
||||
return True
|
||||
|
||||
ts_tests = list(self.base_path.glob("tests/**/*.ts"))
|
||||
for test_file in ts_tests:
|
||||
try:
|
||||
content = test_file.read_text()
|
||||
except OSError:
|
||||
continue
|
||||
if test_name in content:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _get_council_review_count(self, phase_num: int) -> int:
|
||||
@@ -469,7 +496,7 @@ class PhaseValidator:
|
||||
|
||||
try:
|
||||
# Count decisions related to this phase
|
||||
raw = self._redis.lrange("oversight:decisions", 0, 100)
|
||||
raw = cast(list, self._redis.lrange("oversight:decisions", 0, 100))
|
||||
count = 0
|
||||
|
||||
for item in raw:
|
||||
@@ -494,8 +521,8 @@ class PhaseValidator:
|
||||
"status": result.status.value,
|
||||
"coverage": str(result.coverage_percent),
|
||||
"anomalies": str(result.anomalies_found),
|
||||
"validated_at": result.validated_at
|
||||
}
|
||||
"validated_at": result.validated_at,
|
||||
},
|
||||
)
|
||||
|
||||
# Store summary
|
||||
@@ -520,10 +547,14 @@ class PhaseValidator:
|
||||
|
||||
phase_def = self.PHASES.get(phase_num)
|
||||
if phase_def and phase_def.priority == "critical" and result.gaps:
|
||||
critical_gaps.extend([f"Phase {phase_num}: {g}" for g in result.gaps[:2]])
|
||||
critical_gaps.extend(
|
||||
[f"Phase {phase_num}: {g}" for g in result.gaps[:2]]
|
||||
)
|
||||
|
||||
avg_coverage = total_coverage / len(self.results) if self.results else 0
|
||||
|
||||
phase_five = self.results.get(5)
|
||||
|
||||
return {
|
||||
"phases_validated": len(self.results),
|
||||
"by_status": by_status,
|
||||
@@ -531,7 +562,7 @@ class PhaseValidator:
|
||||
"total_anomalies": total_anomalies,
|
||||
"total_gaps": total_gaps,
|
||||
"critical_gaps": critical_gaps[:10],
|
||||
"phase_5_status": self.results.get(5, {}).status.value if self.results.get(5) else "unknown"
|
||||
"phase_5_status": phase_five.status.value if phase_five else "unknown",
|
||||
}
|
||||
|
||||
def get_phase_matrix(self) -> str:
|
||||
@@ -551,7 +582,7 @@ class PhaseValidator:
|
||||
"in_progress": "🚧",
|
||||
"blocked": "❌",
|
||||
"needs_review": "⚠️",
|
||||
"not_started": "⬜"
|
||||
"not_started": "⬜",
|
||||
}
|
||||
|
||||
for phase_num in sorted(self.results.keys()):
|
||||
@@ -566,7 +597,9 @@ class PhaseValidator:
|
||||
lines.append("=" * 80)
|
||||
|
||||
summary = self.get_summary()
|
||||
lines.append(f"Average Coverage: {summary['average_coverage']}% | Anomalies: {summary['total_anomalies']} | Gaps: {summary['total_gaps']}")
|
||||
lines.append(
|
||||
f"Average Coverage: {summary['average_coverage']}% | Anomalies: {summary['total_anomalies']} | Gaps: {summary['total_gaps']}"
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -577,7 +610,11 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Phase Validator")
|
||||
parser.add_argument("command", choices=["validate", "matrix", "phase", "summary"])
|
||||
parser.add_argument("--phase", type=int)
|
||||
parser.add_argument("--level", choices=["basic", "standard", "thorough", "comprehensive"], default="standard")
|
||||
parser.add_argument(
|
||||
"--level",
|
||||
choices=["basic", "standard", "thorough", "comprehensive"],
|
||||
default="standard",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -610,9 +647,9 @@ if __name__ == "__main__":
|
||||
|
||||
elif args.command == "phase" and args.phase:
|
||||
result = validator.validate_phase(args.phase, level)
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"PHASE {result.phase_number}: {result.phase_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Status: {result.status.value}")
|
||||
print(f"Coverage: {result.coverage_percent:.1f}%")
|
||||
print(f"Tests: {result.tests_passed}/{result.tests_found} passed")
|
||||
|
||||
Reference in New Issue
Block a user