Initial commit: Agent Governance System Phase 8
Phase 8 Production Hardening with complete governance infrastructure: - Vault integration with tiered policies (T0-T4) - DragonflyDB state management - SQLite audit ledger - Pipeline DSL and templates - Promotion/revocation engine - Checkpoint system for session persistence - Health manager and circuit breaker for fault tolerance - GitHub/Slack integrations - Architectural test pipeline with bug watcher, suggestion engine, council review - Multi-agent chaos testing framework Test Results: - Governance tests: 68/68 passing - E2E workflow: 16/16 passing - Phase 2 Vault: 14/14 passing - Integration tests: 27/27 passing Coverage: 57.6% average across 12 phases Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
# Runtime
|
||||
|
||||
> Runtime governance and agent lifecycle
|
||||
|
||||
## Overview
|
||||
|
||||
This directory contains runtime governance and agent lifecycle.
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Description |
|
||||
|------|-------------|
|
||||
| `promotion.py` | Python module |
|
||||
| `governance.py` | Python module |
|
||||
| `revocation.py` | Python module |
|
||||
| `monitors.py` | Python module |
|
||||
|
||||
## Interfaces / APIs
|
||||
|
||||
*Document any APIs, CLI commands, or interfaces here.*
|
||||
|
||||
## Status
|
||||
|
||||
** Not Started**
|
||||
|
||||
See [STATUS.md](./STATUS.md) for detailed progress tracking.
|
||||
|
||||
## Architecture Reference
|
||||
|
||||
Part of the [Agent Governance System](/opt/agent-governance/docs/ARCHITECTURE.md).
|
||||
|
||||
Parent: [Project Root](/opt/agent-governance)
|
||||
|
||||
---
|
||||
*Last updated: 2026-01-23 23:25:09 UTC*
|
||||
@@ -0,0 +1,35 @@
|
||||
# Status: Runtime
|
||||
|
||||
## Current Phase
|
||||
|
||||
**COMPLETE**
|
||||
|
||||
## Tasks
|
||||
|
||||
| Status | Task | Updated |
|
||||
|--------|------|---------|
|
||||
| ☐ | *No tasks defined* | - |
|
||||
|
||||
## Dependencies
|
||||
|
||||
*No external dependencies.*
|
||||
|
||||
## Issues / Blockers
|
||||
|
||||
*No current issues or blockers.*
|
||||
|
||||
## Activity Log
|
||||
|
||||
### 2026-01-23 23:25:30 UTC
|
||||
- **Phase**: COMPLETE
|
||||
- **Action**: Governance runtime unified with pipeline.core
|
||||
- **Details**: Phase updated to complete
|
||||
|
||||
### 2026-01-23 23:25:09 UTC
|
||||
- **Phase**: NOT STARTED
|
||||
- **Action**: Initialized
|
||||
- **Details**: Status tracking initialized for this directory.
|
||||
|
||||
|
||||
---
|
||||
*Last updated: 2026-01-23 23:25:30 UTC*
|
||||
@@ -0,0 +1,465 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Circuit Breaker
|
||||
===============
|
||||
Implements circuit breaker pattern for graceful degradation when dependencies fail.
|
||||
|
||||
States:
|
||||
- CLOSED: Normal operation, requests flow through
|
||||
- OPEN: Circuit tripped, requests fail fast
|
||||
- HALF_OPEN: Testing if dependency recovered
|
||||
|
||||
Part of Phase 8: Production Hardening.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, Optional, TypeVar, Generic
|
||||
import redis
|
||||
|
||||
|
||||
class CircuitState(str, Enum):
|
||||
"""Circuit breaker states"""
|
||||
CLOSED = "closed" # Normal operation
|
||||
OPEN = "open" # Failing fast
|
||||
HALF_OPEN = "half_open" # Testing recovery
|
||||
|
||||
|
||||
class FailureType(str, Enum):
|
||||
"""Types of failures tracked"""
|
||||
TIMEOUT = "timeout"
|
||||
CONNECTION_ERROR = "connection_error"
|
||||
EXCEPTION = "exception"
|
||||
THRESHOLD_EXCEEDED = "threshold_exceeded"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CircuitStats:
|
||||
"""Statistics for a circuit breaker"""
|
||||
total_calls: int = 0
|
||||
successful_calls: int = 0
|
||||
failed_calls: int = 0
|
||||
rejected_calls: int = 0
|
||||
last_failure_time: Optional[str] = None
|
||||
last_success_time: Optional[str] = None
|
||||
consecutive_failures: int = 0
|
||||
consecutive_successes: int = 0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"total_calls": self.total_calls,
|
||||
"successful_calls": self.successful_calls,
|
||||
"failed_calls": self.failed_calls,
|
||||
"rejected_calls": self.rejected_calls,
|
||||
"last_failure_time": self.last_failure_time,
|
||||
"last_success_time": self.last_success_time,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"consecutive_successes": self.consecutive_successes,
|
||||
"success_rate": self.success_rate
|
||||
}
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
if self.total_calls == 0:
|
||||
return 1.0
|
||||
return self.successful_calls / self.total_calls
|
||||
|
||||
|
||||
@dataclass
|
||||
class CircuitConfig:
|
||||
"""Configuration for a circuit breaker"""
|
||||
name: str
|
||||
failure_threshold: int = 5 # Failures before opening
|
||||
success_threshold: int = 3 # Successes to close from half-open
|
||||
timeout_seconds: float = 30.0 # Time before trying again (open -> half-open)
|
||||
call_timeout: float = 10.0 # Timeout for individual calls
|
||||
half_open_max_calls: int = 3 # Max concurrent calls in half-open
|
||||
|
||||
|
||||
class CircuitOpenError(Exception):
|
||||
"""Raised when circuit is open and rejecting calls"""
|
||||
def __init__(self, circuit_name: str, retry_after: float):
|
||||
self.circuit_name = circuit_name
|
||||
self.retry_after = retry_after
|
||||
super().__init__(f"Circuit '{circuit_name}' is open. Retry after {retry_after:.1f}s")
|
||||
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
"""
|
||||
Circuit breaker implementation for a single dependency.
|
||||
|
||||
Usage:
|
||||
breaker = CircuitBreaker(CircuitConfig(name="vault"))
|
||||
|
||||
@breaker
|
||||
def call_vault():
|
||||
# ... make vault request
|
||||
pass
|
||||
|
||||
# Or manual:
|
||||
with breaker:
|
||||
# ... protected code
|
||||
"""
|
||||
|
||||
def __init__(self, config: CircuitConfig):
|
||||
self.config = config
|
||||
self._state = CircuitState.CLOSED
|
||||
self._stats = CircuitStats()
|
||||
self._last_failure_time: Optional[float] = None
|
||||
self._half_open_calls = 0
|
||||
self._lock = threading.RLock()
|
||||
|
||||
@property
|
||||
def state(self) -> CircuitState:
|
||||
with self._lock:
|
||||
# Check if we should transition from OPEN to HALF_OPEN
|
||||
if self._state == CircuitState.OPEN:
|
||||
if self._last_failure_time:
|
||||
elapsed = time.time() - self._last_failure_time
|
||||
if elapsed >= self.config.timeout_seconds:
|
||||
self._state = CircuitState.HALF_OPEN
|
||||
self._half_open_calls = 0
|
||||
return self._state
|
||||
|
||||
@property
|
||||
def stats(self) -> CircuitStats:
|
||||
return self._stats
|
||||
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _record_success(self):
|
||||
"""Record a successful call"""
|
||||
with self._lock:
|
||||
self._stats.total_calls += 1
|
||||
self._stats.successful_calls += 1
|
||||
self._stats.consecutive_successes += 1
|
||||
self._stats.consecutive_failures = 0
|
||||
self._stats.last_success_time = self._now()
|
||||
|
||||
if self._state == CircuitState.HALF_OPEN:
|
||||
if self._stats.consecutive_successes >= self.config.success_threshold:
|
||||
self._state = CircuitState.CLOSED
|
||||
self._stats.consecutive_successes = 0
|
||||
|
||||
def _record_failure(self, failure_type: FailureType = FailureType.EXCEPTION):
|
||||
"""Record a failed call"""
|
||||
with self._lock:
|
||||
self._stats.total_calls += 1
|
||||
self._stats.failed_calls += 1
|
||||
self._stats.consecutive_failures += 1
|
||||
self._stats.consecutive_successes = 0
|
||||
self._stats.last_failure_time = self._now()
|
||||
self._last_failure_time = time.time()
|
||||
|
||||
if self._state == CircuitState.CLOSED:
|
||||
if self._stats.consecutive_failures >= self.config.failure_threshold:
|
||||
self._state = CircuitState.OPEN
|
||||
|
||||
elif self._state == CircuitState.HALF_OPEN:
|
||||
# Any failure in half-open goes back to open
|
||||
self._state = CircuitState.OPEN
|
||||
|
||||
def _record_rejection(self):
|
||||
"""Record a rejected call (circuit open)"""
|
||||
with self._lock:
|
||||
self._stats.total_calls += 1
|
||||
self._stats.rejected_calls += 1
|
||||
|
||||
def _can_execute(self) -> bool:
|
||||
"""Check if a call can be executed"""
|
||||
state = self.state # This may transition OPEN -> HALF_OPEN
|
||||
|
||||
if state == CircuitState.CLOSED:
|
||||
return True
|
||||
|
||||
if state == CircuitState.OPEN:
|
||||
return False
|
||||
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
with self._lock:
|
||||
if self._half_open_calls < self.config.half_open_max_calls:
|
||||
self._half_open_calls += 1
|
||||
return True
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def _get_retry_after(self) -> float:
|
||||
"""Get seconds until circuit might close"""
|
||||
if self._last_failure_time:
|
||||
elapsed = time.time() - self._last_failure_time
|
||||
remaining = self.config.timeout_seconds - elapsed
|
||||
return max(0, remaining)
|
||||
return 0
|
||||
|
||||
def execute(self, func: Callable[[], T]) -> T:
|
||||
"""Execute a function with circuit breaker protection"""
|
||||
if not self._can_execute():
|
||||
self._record_rejection()
|
||||
raise CircuitOpenError(self.config.name, self._get_retry_after())
|
||||
|
||||
try:
|
||||
result = func()
|
||||
self._record_success()
|
||||
return result
|
||||
except Exception as e:
|
||||
self._record_failure()
|
||||
raise
|
||||
|
||||
def __call__(self, func: Callable) -> Callable:
|
||||
"""Decorator usage"""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return self.execute(lambda: func(*args, **kwargs))
|
||||
return wrapper
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry"""
|
||||
if not self._can_execute():
|
||||
self._record_rejection()
|
||||
raise CircuitOpenError(self.config.name, self._get_retry_after())
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Context manager exit"""
|
||||
if exc_type is None:
|
||||
self._record_success()
|
||||
else:
|
||||
self._record_failure()
|
||||
return False # Don't suppress exceptions
|
||||
|
||||
def reset(self):
|
||||
"""Manually reset the circuit breaker"""
|
||||
with self._lock:
|
||||
self._state = CircuitState.CLOSED
|
||||
self._stats = CircuitStats()
|
||||
self._last_failure_time = None
|
||||
self._half_open_calls = 0
|
||||
|
||||
def force_open(self):
|
||||
"""Manually open the circuit"""
|
||||
with self._lock:
|
||||
self._state = CircuitState.OPEN
|
||||
self._last_failure_time = time.time()
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Get circuit breaker state as dict"""
|
||||
return {
|
||||
"name": self.config.name,
|
||||
"state": self.state.value,
|
||||
"stats": self._stats.to_dict(),
|
||||
"config": {
|
||||
"failure_threshold": self.config.failure_threshold,
|
||||
"success_threshold": self.config.success_threshold,
|
||||
"timeout_seconds": self.config.timeout_seconds
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class CircuitBreakerRegistry:
|
||||
"""
|
||||
Registry for managing multiple circuit breakers.
|
||||
|
||||
Usage:
|
||||
registry = CircuitBreakerRegistry()
|
||||
registry.register("vault", CircuitConfig(name="vault", failure_threshold=3))
|
||||
|
||||
@registry.protect("vault")
|
||||
def call_vault():
|
||||
# ...
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client: Optional[redis.Redis] = None):
|
||||
self._breakers: dict[str, CircuitBreaker] = {}
|
||||
self._redis = redis_client
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def register(self, name: str, config: Optional[CircuitConfig] = None) -> CircuitBreaker:
|
||||
"""Register a circuit breaker"""
|
||||
with self._lock:
|
||||
if name not in self._breakers:
|
||||
cfg = config or CircuitConfig(name=name)
|
||||
self._breakers[name] = CircuitBreaker(cfg)
|
||||
return self._breakers[name]
|
||||
|
||||
def get(self, name: str) -> Optional[CircuitBreaker]:
|
||||
"""Get a circuit breaker by name"""
|
||||
return self._breakers.get(name)
|
||||
|
||||
def protect(self, name: str) -> Callable:
|
||||
"""Decorator to protect a function with a circuit breaker"""
|
||||
def decorator(func: Callable) -> Callable:
|
||||
breaker = self.register(name)
|
||||
return breaker(func)
|
||||
return decorator
|
||||
|
||||
def get_all_status(self) -> dict:
|
||||
"""Get status of all circuit breakers"""
|
||||
return {
|
||||
name: breaker.to_dict()
|
||||
for name, breaker in self._breakers.items()
|
||||
}
|
||||
|
||||
def reset_all(self):
|
||||
"""Reset all circuit breakers"""
|
||||
for breaker in self._breakers.values():
|
||||
breaker.reset()
|
||||
|
||||
def persist(self):
|
||||
"""Persist circuit breaker states to Redis"""
|
||||
if not self._redis:
|
||||
return
|
||||
|
||||
try:
|
||||
data = json.dumps(self.get_all_status())
|
||||
self._redis.set("circuit_breakers:status", data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Pre-configured Circuit Breakers for Governance System
|
||||
# =============================================================================
|
||||
|
||||
# Global registry
|
||||
_registry = CircuitBreakerRegistry()
|
||||
|
||||
|
||||
def get_vault_breaker() -> CircuitBreaker:
|
||||
"""Get circuit breaker for Vault operations"""
|
||||
return _registry.register("vault", CircuitConfig(
|
||||
name="vault",
|
||||
failure_threshold=3,
|
||||
success_threshold=2,
|
||||
timeout_seconds=30.0,
|
||||
call_timeout=10.0
|
||||
))
|
||||
|
||||
|
||||
def get_dragonfly_breaker() -> CircuitBreaker:
|
||||
"""Get circuit breaker for DragonflyDB operations"""
|
||||
return _registry.register("dragonfly", CircuitConfig(
|
||||
name="dragonfly",
|
||||
failure_threshold=5,
|
||||
success_threshold=3,
|
||||
timeout_seconds=15.0,
|
||||
call_timeout=5.0
|
||||
))
|
||||
|
||||
|
||||
def get_ledger_breaker() -> CircuitBreaker:
|
||||
"""Get circuit breaker for Ledger operations"""
|
||||
return _registry.register("ledger", CircuitConfig(
|
||||
name="ledger",
|
||||
failure_threshold=3,
|
||||
success_threshold=2,
|
||||
timeout_seconds=10.0,
|
||||
call_timeout=5.0
|
||||
))
|
||||
|
||||
|
||||
def get_registry() -> CircuitBreakerRegistry:
|
||||
"""Get the global circuit breaker registry"""
|
||||
return _registry
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Interface
|
||||
# =============================================================================
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Circuit Breaker CLI")
|
||||
parser.add_argument("command", choices=["status", "reset", "test"],
|
||||
help="Command to run")
|
||||
parser.add_argument("--name", type=str, help="Circuit breaker name")
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Initialize default breakers
|
||||
get_vault_breaker()
|
||||
get_dragonfly_breaker()
|
||||
get_ledger_breaker()
|
||||
|
||||
if args.command == "status":
|
||||
status = _registry.get_all_status()
|
||||
if args.json:
|
||||
print(json.dumps(status, indent=2))
|
||||
else:
|
||||
print("\nCircuit Breaker Status")
|
||||
print("=" * 50)
|
||||
for name, data in status.items():
|
||||
state_icon = {
|
||||
"closed": "✅",
|
||||
"open": "❌",
|
||||
"half_open": "⚠️"
|
||||
}.get(data["state"], "?")
|
||||
stats = data["stats"]
|
||||
print(f"\n{state_icon} {name}: {data['state'].upper()}")
|
||||
print(f" Calls: {stats['total_calls']} total, {stats['successful_calls']} success, {stats['failed_calls']} failed")
|
||||
print(f" Success Rate: {stats['success_rate']*100:.1f}%")
|
||||
print(f" Consecutive: {stats['consecutive_failures']} failures, {stats['consecutive_successes']} successes")
|
||||
|
||||
elif args.command == "reset":
|
||||
if args.name:
|
||||
breaker = _registry.get(args.name)
|
||||
if breaker:
|
||||
breaker.reset()
|
||||
print(f"Reset circuit breaker: {args.name}")
|
||||
else:
|
||||
print(f"Circuit breaker not found: {args.name}")
|
||||
else:
|
||||
_registry.reset_all()
|
||||
print("Reset all circuit breakers")
|
||||
|
||||
elif args.command == "test":
|
||||
print("\nTesting circuit breaker behavior...")
|
||||
|
||||
# Create a test breaker
|
||||
test_breaker = CircuitBreaker(CircuitConfig(
|
||||
name="test",
|
||||
failure_threshold=3,
|
||||
success_threshold=2,
|
||||
timeout_seconds=5.0
|
||||
))
|
||||
|
||||
# Simulate failures
|
||||
print("\n1. Simulating 3 failures to trip circuit:")
|
||||
for i in range(3):
|
||||
try:
|
||||
@test_breaker
|
||||
def failing_call():
|
||||
raise Exception("Simulated failure")
|
||||
failing_call()
|
||||
except Exception as e:
|
||||
print(f" Call {i+1}: Failed - {e}")
|
||||
|
||||
print(f"\n Circuit state: {test_breaker.state.value}")
|
||||
|
||||
# Try a call when open
|
||||
print("\n2. Trying call when circuit is open:")
|
||||
try:
|
||||
@test_breaker
|
||||
def blocked_call():
|
||||
return "success"
|
||||
blocked_call()
|
||||
except CircuitOpenError as e:
|
||||
print(f" Rejected: {e}")
|
||||
|
||||
print(f"\n Stats: {test_breaker.stats.to_dict()}")
|
||||
print("\nTest complete!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+712
@@ -0,0 +1,712 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Agent Runtime Governance via DragonflyDB
|
||||
=========================================
|
||||
Implements real-time agent control: instruction packets, state tracking,
|
||||
error budgets, revocation, and handoffs.
|
||||
"""
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
import redis
|
||||
import subprocess
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Configuration
|
||||
# =============================================================================
|
||||
|
||||
def get_dragonfly_client() -> redis.Redis:
|
||||
"""Get DragonflyDB client with credentials from Vault"""
|
||||
# Get credentials from Vault
|
||||
with open("/opt/vault/init-keys.json") as f:
|
||||
token = json.load(f)["root_token"]
|
||||
|
||||
result = subprocess.run([
|
||||
"curl", "-sk",
|
||||
"-H", f"X-Vault-Token: {token}",
|
||||
"https://127.0.0.1:8200/v1/secret/data/services/dragonfly"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
creds = json.loads(result.stdout)["data"]["data"]
|
||||
|
||||
return redis.Redis(
|
||||
host=creds["host"],
|
||||
port=int(creds["port"]),
|
||||
password=creds["password"],
|
||||
decode_responses=True
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Import Core Definitions (from unified pipeline.core module)
|
||||
# =============================================================================
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from pipeline.core import (
|
||||
AgentPhase,
|
||||
AgentStatus,
|
||||
RevocationType,
|
||||
ErrorBudget,
|
||||
AGENT_PHASE_NAMES,
|
||||
AGENT_PHASES_ORDERED,
|
||||
RedisKeys,
|
||||
DEFAULT_HEARTBEAT_TTL,
|
||||
DEFAULT_LOCK_TTL,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class InstructionPacket:
|
||||
agent_id: str
|
||||
task_id: str
|
||||
created_for: str
|
||||
objective: str
|
||||
deliverables: list[str]
|
||||
constraints: dict
|
||||
success_criteria: list[str]
|
||||
error_budget: ErrorBudget
|
||||
escalation_rules: list[str]
|
||||
repo: str = ""
|
||||
pr_context: dict = field(default_factory=dict)
|
||||
revocation_context: dict = field(default_factory=dict)
|
||||
created_at: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.created_at:
|
||||
self.created_at = datetime.now(timezone.utc).isoformat()
|
||||
if isinstance(self.error_budget, dict):
|
||||
self.error_budget = ErrorBudget(**self.error_budget)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = asdict(self)
|
||||
d["error_budget"] = asdict(self.error_budget)
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentState:
|
||||
agent_id: str
|
||||
status: AgentStatus
|
||||
phase: AgentPhase
|
||||
step: str = ""
|
||||
started_at: str = ""
|
||||
last_progress_at: str = ""
|
||||
current_pr: Optional[int] = None
|
||||
notes: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"agent_id": self.agent_id,
|
||||
"status": self.status.value,
|
||||
"phase": self.phase.value,
|
||||
"step": self.step,
|
||||
"started_at": self.started_at,
|
||||
"last_progress_at": self.last_progress_at,
|
||||
"current_pr": self.current_pr,
|
||||
"notes": self.notes
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class HandoffObject:
|
||||
task_id: str
|
||||
previous_agent_id: str
|
||||
revoked: bool
|
||||
revocation_reason: dict
|
||||
last_known_state: dict
|
||||
what_was_tried: list[str]
|
||||
blocking_issue: str
|
||||
required_next_actions: list[str]
|
||||
constraints_reminder: list[str]
|
||||
artifacts: list[str]
|
||||
created_at: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.created_at:
|
||||
self.created_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Governance Manager
|
||||
# =============================================================================
|
||||
|
||||
class GovernanceManager:
|
||||
"""
|
||||
Core runtime governance via DragonflyDB.
|
||||
Manages instruction packets, state, errors, locks, and handoffs.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.db = get_dragonfly_client()
|
||||
self.lock_ttl = 300 # 5 minutes default lock TTL
|
||||
self.heartbeat_ttl = 60 # 1 minute heartbeat timeout
|
||||
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _error_signature(self, error_type: str, message: str) -> str:
|
||||
"""Generate a normalized error signature"""
|
||||
normalized = f"{error_type}:{message[:100]}".lower()
|
||||
return hashlib.md5(normalized.encode()).hexdigest()[:12]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Instruction Packets
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def create_instruction_packet(self, packet: InstructionPacket) -> bool:
|
||||
"""Store an instruction packet for an agent"""
|
||||
key = f"agent:{packet.agent_id}:packet"
|
||||
return self.db.set(key, json.dumps(packet.to_dict()))
|
||||
|
||||
def get_instruction_packet(self, agent_id: str) -> Optional[InstructionPacket]:
|
||||
"""Retrieve an agent's instruction packet"""
|
||||
key = f"agent:{agent_id}:packet"
|
||||
data = self.db.get(key)
|
||||
if data:
|
||||
return InstructionPacket(**json.loads(data))
|
||||
return None
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Agent State
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def set_agent_state(self, state: AgentState) -> bool:
|
||||
"""Update agent's runtime state"""
|
||||
state.last_progress_at = self._now()
|
||||
key = f"agent:{state.agent_id}:state"
|
||||
return self.db.set(key, json.dumps(state.to_dict()))
|
||||
|
||||
def get_agent_state(self, agent_id: str) -> Optional[AgentState]:
|
||||
"""Get agent's current state"""
|
||||
key = f"agent:{agent_id}:state"
|
||||
data = self.db.get(key)
|
||||
if data:
|
||||
d = json.loads(data)
|
||||
return AgentState(
|
||||
agent_id=d["agent_id"],
|
||||
status=AgentStatus(d["status"]),
|
||||
phase=AgentPhase(d["phase"]),
|
||||
step=d.get("step", ""),
|
||||
started_at=d.get("started_at", ""),
|
||||
last_progress_at=d.get("last_progress_at", ""),
|
||||
current_pr=d.get("current_pr"),
|
||||
notes=d.get("notes", "")
|
||||
)
|
||||
return None
|
||||
|
||||
def transition_phase(self, agent_id: str, phase: AgentPhase, step: str = "", notes: str = "") -> bool:
|
||||
"""Transition agent to a new phase"""
|
||||
state = self.get_agent_state(agent_id)
|
||||
if not state:
|
||||
return False
|
||||
|
||||
state.phase = phase
|
||||
state.step = step
|
||||
state.notes = notes
|
||||
return self.set_agent_state(state)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Locking
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def acquire_lock(self, agent_id: str, ttl: int = None) -> bool:
|
||||
"""Acquire execution lock for an agent"""
|
||||
key = f"agent:{agent_id}:lock"
|
||||
ttl = ttl or self.lock_ttl
|
||||
# NX = only set if not exists
|
||||
return self.db.set(key, self._now(), nx=True, ex=ttl)
|
||||
|
||||
def refresh_lock(self, agent_id: str, ttl: int = None) -> bool:
|
||||
"""Refresh an existing lock"""
|
||||
key = f"agent:{agent_id}:lock"
|
||||
ttl = ttl or self.lock_ttl
|
||||
if self.db.exists(key):
|
||||
return self.db.expire(key, ttl)
|
||||
return False
|
||||
|
||||
def release_lock(self, agent_id: str) -> bool:
|
||||
"""Release execution lock"""
|
||||
key = f"agent:{agent_id}:lock"
|
||||
return self.db.delete(key) > 0
|
||||
|
||||
def has_lock(self, agent_id: str) -> bool:
|
||||
"""Check if agent has a valid lock"""
|
||||
key = f"agent:{agent_id}:lock"
|
||||
return self.db.exists(key)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Heartbeat
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def heartbeat(self, agent_id: str) -> bool:
|
||||
"""Update agent heartbeat"""
|
||||
key = f"agent:{agent_id}:heartbeat"
|
||||
return self.db.set(key, self._now(), ex=self.heartbeat_ttl)
|
||||
|
||||
def is_alive(self, agent_id: str) -> bool:
|
||||
"""Check if agent heartbeat is fresh"""
|
||||
key = f"agent:{agent_id}:heartbeat"
|
||||
return self.db.exists(key)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Error Tracking
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def record_error(self, agent_id: str, error_type: str, message: str) -> dict:
|
||||
"""Record an error and return current counts"""
|
||||
key = f"agent:{agent_id}:errors"
|
||||
sig = self._error_signature(error_type, message)
|
||||
|
||||
pipe = self.db.pipeline()
|
||||
pipe.hincrby(key, "total_errors", 1)
|
||||
pipe.hincrby(key, f"same_error:{sig}", 1)
|
||||
pipe.hset(key, "last_error_signature", sig)
|
||||
pipe.hset(key, "last_error_at", self._now())
|
||||
pipe.hset(key, "last_error_type", error_type)
|
||||
pipe.hset(key, "last_error_message", message[:500])
|
||||
pipe.execute()
|
||||
|
||||
return self.get_error_counts(agent_id)
|
||||
|
||||
def record_procedure_violation(self, agent_id: str, violation: str) -> int:
|
||||
"""Record a procedure violation"""
|
||||
key = f"agent:{agent_id}:errors"
|
||||
self.db.hset(key, "last_violation", violation)
|
||||
self.db.hset(key, "last_violation_at", self._now())
|
||||
return self.db.hincrby(key, "procedure_violations", 1)
|
||||
|
||||
def get_error_counts(self, agent_id: str) -> dict:
|
||||
"""Get all error counts for an agent"""
|
||||
key = f"agent:{agent_id}:errors"
|
||||
data = self.db.hgetall(key)
|
||||
return {
|
||||
"total_errors": int(data.get("total_errors", 0)),
|
||||
"procedure_violations": int(data.get("procedure_violations", 0)),
|
||||
"last_error_signature": data.get("last_error_signature", ""),
|
||||
"last_error_at": data.get("last_error_at", ""),
|
||||
"last_error_type": data.get("last_error_type", ""),
|
||||
"same_error_counts": {
|
||||
k.replace("same_error:", ""): int(v)
|
||||
for k, v in data.items()
|
||||
if k.startswith("same_error:")
|
||||
}
|
||||
}
|
||||
|
||||
def check_error_budget(self, agent_id: str) -> tuple[bool, Optional[str]]:
|
||||
"""Check if error budget is exceeded. Returns (ok, reason)"""
|
||||
packet = self.get_instruction_packet(agent_id)
|
||||
if not packet:
|
||||
return False, "NO_INSTRUCTION_PACKET"
|
||||
|
||||
counts = self.get_error_counts(agent_id)
|
||||
budget = packet.error_budget
|
||||
|
||||
# Check procedure violations
|
||||
if counts["procedure_violations"] >= budget.max_procedure_violations:
|
||||
return False, f"PROCEDURE_VIOLATIONS ({counts['procedure_violations']} >= {budget.max_procedure_violations})"
|
||||
|
||||
# Check total errors
|
||||
if counts["total_errors"] >= budget.max_total_errors:
|
||||
return False, f"TOTAL_ERRORS ({counts['total_errors']} >= {budget.max_total_errors})"
|
||||
|
||||
# Check same error repeats
|
||||
for sig, count in counts["same_error_counts"].items():
|
||||
if count >= budget.max_same_error_repeats:
|
||||
return False, f"SAME_ERROR_REPEATED ({sig}: {count} >= {budget.max_same_error_repeats})"
|
||||
|
||||
return True, None
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Task Management
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def assign_agent_to_task(self, task_id: str, agent_id: str) -> bool:
|
||||
"""Assign an agent to a task"""
|
||||
# Set active agent
|
||||
self.db.set(f"task:{task_id}:active_agent", agent_id)
|
||||
# Add to history
|
||||
self.db.rpush(f"task:{task_id}:history", json.dumps({
|
||||
"agent_id": agent_id,
|
||||
"assigned_at": self._now(),
|
||||
"event": "ASSIGNED"
|
||||
}))
|
||||
return True
|
||||
|
||||
def get_active_agent(self, task_id: str) -> Optional[str]:
|
||||
"""Get the currently active agent for a task"""
|
||||
return self.db.get(f"task:{task_id}:active_agent")
|
||||
|
||||
def get_task_history(self, task_id: str) -> list[dict]:
|
||||
"""Get task agent history"""
|
||||
data = self.db.lrange(f"task:{task_id}:history", 0, -1)
|
||||
return [json.loads(d) for d in data]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Revocation
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def revoke_agent(self, agent_id: str, reason_type: RevocationType, details: str) -> bool:
|
||||
"""Revoke an agent's access"""
|
||||
# Update state
|
||||
state = self.get_agent_state(agent_id)
|
||||
if state:
|
||||
state.status = AgentStatus.REVOKED
|
||||
state.phase = AgentPhase.REVOKED
|
||||
state.notes = f"Revoked: {reason_type.value} - {details}"
|
||||
self.set_agent_state(state)
|
||||
|
||||
# Release lock
|
||||
self.release_lock(agent_id)
|
||||
|
||||
# Write to revocation ledger
|
||||
revocation_event = {
|
||||
"agent_id": agent_id,
|
||||
"reason_type": reason_type.value,
|
||||
"details": details,
|
||||
"revoked_at": self._now()
|
||||
}
|
||||
self.db.rpush("revocations:ledger", json.dumps(revocation_event))
|
||||
|
||||
# Add to task history if we know the task
|
||||
packet = self.get_instruction_packet(agent_id)
|
||||
if packet:
|
||||
self.db.rpush(f"task:{packet.task_id}:history", json.dumps({
|
||||
"agent_id": agent_id,
|
||||
"event": "REVOKED",
|
||||
"reason": reason_type.value,
|
||||
"revoked_at": self._now()
|
||||
}))
|
||||
|
||||
return True
|
||||
|
||||
def get_recent_revocations(self, count: int = 50) -> list[dict]:
|
||||
"""Get recent revocation events"""
|
||||
data = self.db.lrange("revocations:ledger", -count, -1)
|
||||
return [json.loads(d) for d in data]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Handoff
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def create_handoff(self, handoff: HandoffObject) -> bool:
|
||||
"""Create a handoff object for task continuity"""
|
||||
key = f"handoff:{handoff.task_id}:latest"
|
||||
return self.db.set(key, json.dumps(handoff.to_dict()))
|
||||
|
||||
def get_handoff(self, task_id: str) -> Optional[HandoffObject]:
|
||||
"""Get the latest handoff for a task"""
|
||||
key = f"handoff:{task_id}:latest"
|
||||
data = self.db.get(key)
|
||||
if data:
|
||||
return HandoffObject(**json.loads(data))
|
||||
return None
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Artifacts
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def register_artifact(self, task_id: str, artifact_type: str, reference: str) -> bool:
|
||||
"""Register an artifact for a task"""
|
||||
key = f"task:{task_id}:artifacts"
|
||||
artifact = {
|
||||
"type": artifact_type,
|
||||
"reference": reference,
|
||||
"created_at": self._now()
|
||||
}
|
||||
self.db.rpush(key, json.dumps(artifact))
|
||||
return True
|
||||
|
||||
def get_artifacts(self, task_id: str) -> list[dict]:
|
||||
"""Get all artifacts for a task"""
|
||||
key = f"task:{task_id}:artifacts"
|
||||
data = self.db.lrange(key, 0, -1)
|
||||
return [json.loads(d) for d in data]
|
||||
|
||||
def has_required_artifact(self, task_id: str, artifact_type: str) -> bool:
|
||||
"""Check if a required artifact exists"""
|
||||
artifacts = self.get_artifacts(task_id)
|
||||
return any(a["type"] == artifact_type for a in artifacts)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Worker Agent Runtime
|
||||
# =============================================================================
|
||||
|
||||
class WorkerRuntime:
|
||||
"""
|
||||
Runtime for worker agents.
|
||||
Handles bootstrap, state transitions, error reporting, and compliance.
|
||||
"""
|
||||
|
||||
def __init__(self, agent_id: str):
|
||||
self.agent_id = agent_id
|
||||
self.gov = GovernanceManager()
|
||||
self.packet: Optional[InstructionPacket] = None
|
||||
self.state: Optional[AgentState] = None
|
||||
|
||||
def bootstrap(self) -> tuple[bool, str]:
|
||||
"""
|
||||
Bootstrap the agent runtime.
|
||||
Returns (success, message)
|
||||
"""
|
||||
# Step 1: Read revocation ledger
|
||||
revocations = self.gov.get_recent_revocations(50)
|
||||
if revocations:
|
||||
print(f"[BOOTSTRAP] Read {len(revocations)} recent revocations")
|
||||
# Check if this agent was previously revoked
|
||||
for rev in revocations:
|
||||
if rev["agent_id"] == self.agent_id:
|
||||
return False, f"AGENT_PREVIOUSLY_REVOKED: {rev['reason_type']}"
|
||||
|
||||
# Step 2: Read instruction packet
|
||||
self.packet = self.gov.get_instruction_packet(self.agent_id)
|
||||
if not self.packet:
|
||||
return False, "NO_INSTRUCTION_PACKET"
|
||||
|
||||
print(f"[BOOTSTRAP] Loaded instruction packet for task: {self.packet.task_id}")
|
||||
|
||||
# Step 3: Check for existing handoff
|
||||
handoff = self.gov.get_handoff(self.packet.task_id)
|
||||
if handoff and handoff.previous_agent_id != self.agent_id:
|
||||
print(f"[BOOTSTRAP] Found handoff from previous agent: {handoff.previous_agent_id}")
|
||||
print(f"[BOOTSTRAP] Revocation reason: {handoff.revocation_reason}")
|
||||
print(f"[BOOTSTRAP] Required next actions: {handoff.required_next_actions}")
|
||||
|
||||
# Step 4: Acquire lock
|
||||
if not self.gov.acquire_lock(self.agent_id):
|
||||
return False, "CANNOT_ACQUIRE_LOCK"
|
||||
|
||||
# Step 5: Initialize state
|
||||
self.state = AgentState(
|
||||
agent_id=self.agent_id,
|
||||
status=AgentStatus.RUNNING,
|
||||
phase=AgentPhase.BOOTSTRAP,
|
||||
step="initialized",
|
||||
started_at=datetime.now(timezone.utc).isoformat()
|
||||
)
|
||||
self.gov.set_agent_state(self.state)
|
||||
|
||||
# Step 6: Start heartbeat
|
||||
self.gov.heartbeat(self.agent_id)
|
||||
|
||||
# Step 7: Assign to task
|
||||
self.gov.assign_agent_to_task(self.packet.task_id, self.agent_id)
|
||||
|
||||
return True, "BOOTSTRAP_COMPLETE"
|
||||
|
||||
def transition(self, phase: AgentPhase, step: str = "", notes: str = "") -> bool:
|
||||
"""Transition to a new phase"""
|
||||
if not self.state:
|
||||
return False
|
||||
|
||||
# Refresh heartbeat and lock
|
||||
self.gov.heartbeat(self.agent_id)
|
||||
self.gov.refresh_lock(self.agent_id)
|
||||
|
||||
# Check error budget before transition
|
||||
ok, reason = self.gov.check_error_budget(self.agent_id)
|
||||
if not ok:
|
||||
self.gov.revoke_agent(self.agent_id, RevocationType.ERROR_BUDGET_EXCEEDED, reason)
|
||||
return False
|
||||
|
||||
# Update state
|
||||
self.state.phase = phase
|
||||
self.state.step = step
|
||||
self.state.notes = notes
|
||||
self.gov.set_agent_state(self.state)
|
||||
|
||||
print(f"[PHASE] {phase.value} - {step}")
|
||||
return True
|
||||
|
||||
def report_error(self, error_type: str, message: str) -> bool:
|
||||
"""Report an error and check if we should continue"""
|
||||
counts = self.gov.record_error(self.agent_id, error_type, message)
|
||||
print(f"[ERROR] {error_type}: {message}")
|
||||
print(f"[ERROR] Counts: total={counts['total_errors']}, violations={counts['procedure_violations']}")
|
||||
|
||||
ok, reason = self.gov.check_error_budget(self.agent_id)
|
||||
if not ok:
|
||||
print(f"[REVOKE] Error budget exceeded: {reason}")
|
||||
self.gov.revoke_agent(self.agent_id, RevocationType.ERROR_BUDGET_EXCEEDED, reason)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def report_violation(self, violation: str) -> bool:
|
||||
"""Report a procedure violation"""
|
||||
count = self.gov.record_procedure_violation(self.agent_id, violation)
|
||||
print(f"[VIOLATION] {violation} (count: {count})")
|
||||
|
||||
ok, reason = self.gov.check_error_budget(self.agent_id)
|
||||
if not ok:
|
||||
print(f"[REVOKE] Procedure violation limit: {reason}")
|
||||
self.gov.revoke_agent(self.agent_id, RevocationType.PROCEDURE_VIOLATION, reason)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def register_artifact(self, artifact_type: str, reference: str):
|
||||
"""Register a work artifact"""
|
||||
if self.packet:
|
||||
self.gov.register_artifact(self.packet.task_id, artifact_type, reference)
|
||||
print(f"[ARTIFACT] Registered: {artifact_type} -> {reference}")
|
||||
|
||||
def complete(self, notes: str = "") -> bool:
|
||||
"""Mark work as complete"""
|
||||
if self.state:
|
||||
self.state.status = AgentStatus.COMPLETED
|
||||
self.state.phase = AgentPhase.EXIT
|
||||
self.state.notes = notes
|
||||
self.gov.set_agent_state(self.state)
|
||||
|
||||
self.gov.release_lock(self.agent_id)
|
||||
print(f"[COMPLETE] {notes}")
|
||||
return True
|
||||
|
||||
def create_handoff(self, blocking_issue: str, what_was_tried: list[str],
|
||||
required_next_actions: list[str]) -> bool:
|
||||
"""Create a handoff for the next agent"""
|
||||
if not self.packet or not self.state:
|
||||
return False
|
||||
|
||||
handoff = HandoffObject(
|
||||
task_id=self.packet.task_id,
|
||||
previous_agent_id=self.agent_id,
|
||||
revoked=self.state.status == AgentStatus.REVOKED,
|
||||
revocation_reason={
|
||||
"type": self.state.status.value,
|
||||
"details": self.state.notes
|
||||
},
|
||||
last_known_state={
|
||||
"phase": self.state.phase.value,
|
||||
"step": self.state.step
|
||||
},
|
||||
what_was_tried=what_was_tried,
|
||||
blocking_issue=blocking_issue,
|
||||
required_next_actions=required_next_actions,
|
||||
constraints_reminder=self.packet.constraints.get("forbidden", []),
|
||||
artifacts=[a["reference"] for a in self.gov.get_artifacts(self.packet.task_id)]
|
||||
)
|
||||
|
||||
return self.gov.create_handoff(handoff)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
gov = GovernanceManager()
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: governance.py <command> [args]")
|
||||
print("Commands:")
|
||||
print(" create-packet <agent_id> <task_id> <objective>")
|
||||
print(" get-state <agent_id>")
|
||||
print(" get-errors <agent_id>")
|
||||
print(" revocations")
|
||||
print(" test")
|
||||
sys.exit(1)
|
||||
|
||||
cmd = sys.argv[1]
|
||||
|
||||
if cmd == "create-packet":
|
||||
agent_id, task_id, objective = sys.argv[2], sys.argv[3], sys.argv[4]
|
||||
packet = InstructionPacket(
|
||||
agent_id=agent_id,
|
||||
task_id=task_id,
|
||||
created_for="CLI test",
|
||||
objective=objective,
|
||||
deliverables=["plan artifacts", "run logs"],
|
||||
constraints={
|
||||
"scope": ["sandbox only"],
|
||||
"forbidden": ["no prod access", "no unrecorded root"],
|
||||
"required_steps": ["plan before apply"]
|
||||
},
|
||||
success_criteria=["CI passes", "artifacts uploaded"],
|
||||
error_budget=ErrorBudget(),
|
||||
escalation_rules=["If blocked > 20m -> escalate"]
|
||||
)
|
||||
gov.create_instruction_packet(packet)
|
||||
print(f"Created packet for {agent_id}")
|
||||
|
||||
elif cmd == "get-state":
|
||||
agent_id = sys.argv[2]
|
||||
state = gov.get_agent_state(agent_id)
|
||||
if state:
|
||||
print(json.dumps(state.to_dict(), indent=2))
|
||||
else:
|
||||
print("No state found")
|
||||
|
||||
elif cmd == "get-errors":
|
||||
agent_id = sys.argv[2]
|
||||
errors = gov.get_error_counts(agent_id)
|
||||
print(json.dumps(errors, indent=2))
|
||||
|
||||
elif cmd == "revocations":
|
||||
revs = gov.get_recent_revocations()
|
||||
for r in revs:
|
||||
print(json.dumps(r))
|
||||
|
||||
elif cmd == "test":
|
||||
print("=== Testing Governance System ===\n")
|
||||
|
||||
# Create test packet
|
||||
packet = InstructionPacket(
|
||||
agent_id="test-agent-001",
|
||||
task_id="test-task-001",
|
||||
created_for="Governance test",
|
||||
objective="Test the governance system",
|
||||
deliverables=["test output"],
|
||||
constraints={
|
||||
"scope": ["sandbox"],
|
||||
"forbidden": ["prod access"],
|
||||
"required_steps": ["plan first"]
|
||||
},
|
||||
success_criteria=["test passes"],
|
||||
error_budget=ErrorBudget(max_total_errors=5, max_same_error_repeats=2),
|
||||
escalation_rules=[]
|
||||
)
|
||||
gov.create_instruction_packet(packet)
|
||||
print("[OK] Created instruction packet")
|
||||
|
||||
# Test worker runtime
|
||||
worker = WorkerRuntime("test-agent-001")
|
||||
ok, msg = worker.bootstrap()
|
||||
print(f"[OK] Bootstrap: {msg}")
|
||||
|
||||
# Simulate work phases
|
||||
worker.transition(AgentPhase.PREFLIGHT, "scope_check")
|
||||
worker.transition(AgentPhase.PLAN, "generating_plan")
|
||||
worker.register_artifact("plan", "plan_output_001")
|
||||
worker.transition(AgentPhase.EXECUTE, "applying")
|
||||
|
||||
# Simulate some errors
|
||||
worker.report_error("VALIDATION_ERROR", "Variable not defined")
|
||||
worker.report_error("VALIDATION_ERROR", "Variable not defined") # Same error
|
||||
|
||||
# Check error budget
|
||||
ok, reason = gov.check_error_budget("test-agent-001")
|
||||
print(f"[OK] Error budget check: ok={ok}, reason={reason}")
|
||||
|
||||
# Complete
|
||||
worker.complete("Test completed successfully")
|
||||
|
||||
print("\n=== Test Complete ===")
|
||||
|
||||
else:
|
||||
print(f"Unknown command: {cmd}")
|
||||
@@ -0,0 +1,448 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Health Manager
|
||||
==============
|
||||
Production health check infrastructure for monitoring system dependencies.
|
||||
|
||||
Monitors:
|
||||
- HashiCorp Vault (authentication, policies)
|
||||
- DragonflyDB (state management)
|
||||
- SQLite Ledger (audit trail)
|
||||
- Agent processes
|
||||
|
||||
Part of Phase 8: Production Hardening.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Optional, Callable
|
||||
import redis
|
||||
|
||||
|
||||
class HealthStatus(str, Enum):
|
||||
"""Health check status"""
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
UNHEALTHY = "unhealthy"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class DependencyType(str, Enum):
|
||||
"""Types of system dependencies"""
|
||||
VAULT = "vault"
|
||||
DRAGONFLY = "dragonfly"
|
||||
LEDGER = "ledger"
|
||||
AGENT = "agent"
|
||||
NETWORK = "network"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthCheckResult:
|
||||
"""Result of a health check"""
|
||||
dependency: DependencyType
|
||||
status: HealthStatus
|
||||
latency_ms: float
|
||||
message: str
|
||||
checked_at: str = ""
|
||||
details: dict = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.checked_at:
|
||||
self.checked_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"dependency": self.dependency.value,
|
||||
"status": self.status.value,
|
||||
"latency_ms": self.latency_ms,
|
||||
"message": self.message,
|
||||
"checked_at": self.checked_at,
|
||||
"details": self.details
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemHealth:
|
||||
"""Overall system health status"""
|
||||
status: HealthStatus
|
||||
checks: list[HealthCheckResult]
|
||||
healthy_count: int
|
||||
degraded_count: int
|
||||
unhealthy_count: int
|
||||
checked_at: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.checked_at:
|
||||
self.checked_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"status": self.status.value,
|
||||
"healthy": self.healthy_count,
|
||||
"degraded": self.degraded_count,
|
||||
"unhealthy": self.unhealthy_count,
|
||||
"checked_at": self.checked_at,
|
||||
"checks": [c.to_dict() for c in self.checks]
|
||||
}
|
||||
|
||||
|
||||
class HealthManager:
|
||||
"""
|
||||
Manages health checks for all system dependencies.
|
||||
|
||||
Features:
|
||||
- Individual dependency health checks
|
||||
- Aggregate system health
|
||||
- Configurable thresholds
|
||||
- Health history tracking
|
||||
"""
|
||||
|
||||
# Default configuration
|
||||
VAULT_ADDR = "https://127.0.0.1:8200"
|
||||
VAULT_TOKEN_FILE = "/opt/vault/init-keys.json"
|
||||
REDIS_HOST = "127.0.0.1"
|
||||
REDIS_PORT = 6379
|
||||
LEDGER_PATH = "/opt/agent-governance/ledger/governance.db"
|
||||
|
||||
# Thresholds (ms)
|
||||
LATENCY_WARN_THRESHOLD = 500
|
||||
LATENCY_CRITICAL_THRESHOLD = 2000
|
||||
CHECK_TIMEOUT = 5
|
||||
|
||||
def __init__(self, base_path: str = "/opt/agent-governance"):
|
||||
self.base_path = Path(base_path)
|
||||
self._vault_token: Optional[str] = None
|
||||
self._redis: Optional[redis.Redis] = None
|
||||
self._history: list[SystemHealth] = []
|
||||
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _get_vault_token(self) -> str:
|
||||
"""Get Vault root token"""
|
||||
if self._vault_token:
|
||||
return self._vault_token
|
||||
try:
|
||||
with open(self.VAULT_TOKEN_FILE) as f:
|
||||
self._vault_token = json.load(f)["root_token"]
|
||||
return self._vault_token
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _get_redis(self) -> Optional[redis.Redis]:
|
||||
"""Get Redis/DragonflyDB client"""
|
||||
if self._redis:
|
||||
return self._redis
|
||||
try:
|
||||
# Get credentials from Vault
|
||||
token = self._get_vault_token()
|
||||
result = subprocess.run([
|
||||
"curl", "-sk",
|
||||
"-H", f"X-Vault-Token: {token}",
|
||||
f"{self.VAULT_ADDR}/v1/secret/data/services/dragonfly"
|
||||
], capture_output=True, text=True, timeout=self.CHECK_TIMEOUT)
|
||||
|
||||
creds = json.loads(result.stdout)["data"]["data"]
|
||||
self._redis = redis.Redis(
|
||||
host=creds["host"],
|
||||
port=int(creds["port"]),
|
||||
password=creds["password"],
|
||||
decode_responses=True,
|
||||
socket_timeout=self.CHECK_TIMEOUT
|
||||
)
|
||||
return self._redis
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def check_vault(self) -> HealthCheckResult:
|
||||
"""Check Vault health"""
|
||||
start = time.time()
|
||||
try:
|
||||
token = self._get_vault_token()
|
||||
if not token:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.VAULT,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=0,
|
||||
message="Cannot read Vault token"
|
||||
)
|
||||
|
||||
result = subprocess.run([
|
||||
"curl", "-sk",
|
||||
"-H", f"X-Vault-Token: {token}",
|
||||
f"{self.VAULT_ADDR}/v1/sys/health"
|
||||
], capture_output=True, text=True, timeout=self.CHECK_TIMEOUT)
|
||||
|
||||
latency = (time.time() - start) * 1000
|
||||
data = json.loads(result.stdout)
|
||||
|
||||
if data.get("sealed"):
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.VAULT,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=latency,
|
||||
message="Vault is sealed",
|
||||
details=data
|
||||
)
|
||||
|
||||
if not data.get("initialized"):
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.VAULT,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=latency,
|
||||
message="Vault is not initialized",
|
||||
details=data
|
||||
)
|
||||
|
||||
status = HealthStatus.HEALTHY
|
||||
if latency > self.LATENCY_CRITICAL_THRESHOLD:
|
||||
status = HealthStatus.DEGRADED
|
||||
elif latency > self.LATENCY_WARN_THRESHOLD:
|
||||
status = HealthStatus.DEGRADED
|
||||
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.VAULT,
|
||||
status=status,
|
||||
latency_ms=latency,
|
||||
message="Vault is healthy",
|
||||
details={"initialized": True, "sealed": False}
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.VAULT,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=self.CHECK_TIMEOUT * 1000,
|
||||
message="Vault health check timed out"
|
||||
)
|
||||
except Exception as e:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.VAULT,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=(time.time() - start) * 1000,
|
||||
message=f"Vault check failed: {str(e)}"
|
||||
)
|
||||
|
||||
def check_dragonfly(self) -> HealthCheckResult:
|
||||
"""Check DragonflyDB health"""
|
||||
start = time.time()
|
||||
try:
|
||||
r = self._get_redis()
|
||||
if not r:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.DRAGONFLY,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=0,
|
||||
message="Cannot connect to DragonflyDB"
|
||||
)
|
||||
|
||||
# Ping test
|
||||
r.ping()
|
||||
latency = (time.time() - start) * 1000
|
||||
|
||||
# Get info
|
||||
info = r.info("server")
|
||||
|
||||
status = HealthStatus.HEALTHY
|
||||
if latency > self.LATENCY_CRITICAL_THRESHOLD:
|
||||
status = HealthStatus.DEGRADED
|
||||
elif latency > self.LATENCY_WARN_THRESHOLD:
|
||||
status = HealthStatus.DEGRADED
|
||||
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.DRAGONFLY,
|
||||
status=status,
|
||||
latency_ms=latency,
|
||||
message="DragonflyDB is healthy",
|
||||
details={
|
||||
"version": info.get("redis_version", "unknown"),
|
||||
"uptime_seconds": info.get("uptime_in_seconds", 0)
|
||||
}
|
||||
)
|
||||
|
||||
except redis.ConnectionError:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.DRAGONFLY,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=(time.time() - start) * 1000,
|
||||
message="DragonflyDB connection refused"
|
||||
)
|
||||
except Exception as e:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.DRAGONFLY,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=(time.time() - start) * 1000,
|
||||
message=f"DragonflyDB check failed: {str(e)}"
|
||||
)
|
||||
|
||||
def check_ledger(self) -> HealthCheckResult:
|
||||
"""Check SQLite Ledger health"""
|
||||
start = time.time()
|
||||
try:
|
||||
if not Path(self.LEDGER_PATH).exists():
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.LEDGER,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=0,
|
||||
message="Ledger database file not found"
|
||||
)
|
||||
|
||||
conn = sqlite3.connect(self.LEDGER_PATH, timeout=self.CHECK_TIMEOUT)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check tables exist
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
tables = [row[0] for row in cursor.fetchall()]
|
||||
required_tables = ["agent_metrics", "violations", "promotions", "orchestration_log"]
|
||||
|
||||
missing = [t for t in required_tables if t not in tables]
|
||||
|
||||
latency = (time.time() - start) * 1000
|
||||
conn.close()
|
||||
|
||||
if missing:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.LEDGER,
|
||||
status=HealthStatus.DEGRADED,
|
||||
latency_ms=latency,
|
||||
message=f"Missing tables: {', '.join(missing)}",
|
||||
details={"tables": tables, "missing": missing}
|
||||
)
|
||||
|
||||
status = HealthStatus.HEALTHY
|
||||
if latency > self.LATENCY_WARN_THRESHOLD:
|
||||
status = HealthStatus.DEGRADED
|
||||
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.LEDGER,
|
||||
status=status,
|
||||
latency_ms=latency,
|
||||
message="Ledger is healthy",
|
||||
details={"tables": tables}
|
||||
)
|
||||
|
||||
except sqlite3.OperationalError as e:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.LEDGER,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=(time.time() - start) * 1000,
|
||||
message=f"Ledger error: {str(e)}"
|
||||
)
|
||||
except Exception as e:
|
||||
return HealthCheckResult(
|
||||
dependency=DependencyType.LEDGER,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
latency_ms=(time.time() - start) * 1000,
|
||||
message=f"Ledger check failed: {str(e)}"
|
||||
)
|
||||
|
||||
def check_all(self) -> SystemHealth:
|
||||
"""Run all health checks and return aggregate status"""
|
||||
checks = [
|
||||
self.check_vault(),
|
||||
self.check_dragonfly(),
|
||||
self.check_ledger()
|
||||
]
|
||||
|
||||
healthy = sum(1 for c in checks if c.status == HealthStatus.HEALTHY)
|
||||
degraded = sum(1 for c in checks if c.status == HealthStatus.DEGRADED)
|
||||
unhealthy = sum(1 for c in checks if c.status == HealthStatus.UNHEALTHY)
|
||||
|
||||
# Determine overall status
|
||||
if unhealthy > 0:
|
||||
overall = HealthStatus.UNHEALTHY
|
||||
elif degraded > 0:
|
||||
overall = HealthStatus.DEGRADED
|
||||
else:
|
||||
overall = HealthStatus.HEALTHY
|
||||
|
||||
health = SystemHealth(
|
||||
status=overall,
|
||||
checks=checks,
|
||||
healthy_count=healthy,
|
||||
degraded_count=degraded,
|
||||
unhealthy_count=unhealthy
|
||||
)
|
||||
|
||||
# Track history
|
||||
self._history.append(health)
|
||||
if len(self._history) > 100:
|
||||
self._history = self._history[-100:]
|
||||
|
||||
# Store in DragonflyDB
|
||||
try:
|
||||
r = self._get_redis()
|
||||
if r:
|
||||
r.set("health:latest", json.dumps(health.to_dict()))
|
||||
r.lpush("health:history", json.dumps(health.to_dict()))
|
||||
r.ltrim("health:history", 0, 99)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return health
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""Get current health status as dict"""
|
||||
health = self.check_all()
|
||||
return health.to_dict()
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""Quick check if system is healthy"""
|
||||
health = self.check_all()
|
||||
return health.status == HealthStatus.HEALTHY
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Interface
|
||||
# =============================================================================
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Health Manager CLI")
|
||||
parser.add_argument("command", choices=["check", "status", "watch"],
|
||||
help="Command to run")
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
parser.add_argument("--interval", type=int, default=5,
|
||||
help="Watch interval in seconds")
|
||||
|
||||
args = parser.parse_args()
|
||||
manager = HealthManager()
|
||||
|
||||
if args.command == "check":
|
||||
health = manager.check_all()
|
||||
if args.json:
|
||||
print(json.dumps(health.to_dict(), indent=2))
|
||||
else:
|
||||
print(f"\nSystem Health: {health.status.value.upper()}")
|
||||
print(f"Healthy: {health.healthy_count} | Degraded: {health.degraded_count} | Unhealthy: {health.unhealthy_count}")
|
||||
print()
|
||||
for check in health.checks:
|
||||
icon = {"healthy": "✅", "degraded": "⚠️", "unhealthy": "❌"}.get(check.status.value, "?")
|
||||
print(f" {icon} {check.dependency.value}: {check.message} ({check.latency_ms:.1f}ms)")
|
||||
|
||||
elif args.command == "status":
|
||||
health = manager.check_all()
|
||||
print("HEALTHY" if health.status == HealthStatus.HEALTHY else health.status.value.upper())
|
||||
|
||||
elif args.command == "watch":
|
||||
print("Watching health (Ctrl+C to stop)...")
|
||||
try:
|
||||
while True:
|
||||
health = manager.check_all()
|
||||
timestamp = datetime.now().strftime("%H:%M:%S")
|
||||
status_icon = {"healthy": "✅", "degraded": "⚠️", "unhealthy": "❌"}.get(health.status.value, "?")
|
||||
print(f"[{timestamp}] {status_icon} {health.status.value} - V:{health.checks[0].latency_ms:.0f}ms D:{health.checks[1].latency_ms:.0f}ms L:{health.checks[2].latency_ms:.0f}ms")
|
||||
time.sleep(args.interval)
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopped.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+317
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Governance Monitor Agents
|
||||
=========================
|
||||
These agents watch and enforce, they don't do work.
|
||||
|
||||
- Execution Watcher: Heartbeats, stuck detection, lock expiry
|
||||
- Compliance Watcher: Artifact checks, forbidden action detection
|
||||
- Vault Lease Watcher: Token validity, revocation enforcement
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
import threading
|
||||
|
||||
from governance import (
|
||||
GovernanceManager,
|
||||
AgentPhase,
|
||||
AgentStatus,
|
||||
RevocationType
|
||||
)
|
||||
|
||||
|
||||
class ExecutionWatcher:
|
||||
"""
|
||||
Monitors agent execution health.
|
||||
- Checks heartbeats
|
||||
- Detects stuck agents
|
||||
- Handles lock expiry
|
||||
"""
|
||||
|
||||
def __init__(self, heartbeat_timeout: int = 60, stuck_threshold: int = 300):
|
||||
self.gov = GovernanceManager()
|
||||
self.heartbeat_timeout = heartbeat_timeout # seconds
|
||||
self.stuck_threshold = stuck_threshold # seconds without progress
|
||||
|
||||
def check_agent(self, agent_id: str) -> dict:
|
||||
"""Check an agent's execution health"""
|
||||
result = {
|
||||
"agent_id": agent_id,
|
||||
"healthy": True,
|
||||
"issues": []
|
||||
}
|
||||
|
||||
state = self.gov.get_agent_state(agent_id)
|
||||
if not state:
|
||||
result["healthy"] = False
|
||||
result["issues"].append("NO_STATE")
|
||||
return result
|
||||
|
||||
# Skip completed/revoked agents
|
||||
if state.status in [AgentStatus.COMPLETED, AgentStatus.REVOKED]:
|
||||
return result
|
||||
|
||||
# Check heartbeat
|
||||
if not self.gov.is_alive(agent_id):
|
||||
result["healthy"] = False
|
||||
result["issues"].append("HEARTBEAT_TIMEOUT")
|
||||
|
||||
# Check lock
|
||||
if not self.gov.has_lock(agent_id) and state.status == AgentStatus.RUNNING:
|
||||
result["healthy"] = False
|
||||
result["issues"].append("LOCK_EXPIRED")
|
||||
|
||||
# Check for stuck (no progress)
|
||||
if state.last_progress_at:
|
||||
last_progress = datetime.fromisoformat(state.last_progress_at.replace("Z", "+00:00"))
|
||||
age = (datetime.now(timezone.utc) - last_progress).total_seconds()
|
||||
if age > self.stuck_threshold:
|
||||
result["healthy"] = False
|
||||
result["issues"].append(f"STUCK_{int(age)}s")
|
||||
|
||||
return result
|
||||
|
||||
def enforce(self, agent_id: str) -> Optional[str]:
|
||||
"""Enforce health requirements, return action taken"""
|
||||
check = self.check_agent(agent_id)
|
||||
|
||||
if check["healthy"]:
|
||||
return None
|
||||
|
||||
for issue in check["issues"]:
|
||||
if issue == "HEARTBEAT_TIMEOUT":
|
||||
self.gov.revoke_agent(agent_id, RevocationType.HEARTBEAT_TIMEOUT,
|
||||
"No heartbeat received within timeout")
|
||||
return "REVOKED:HEARTBEAT"
|
||||
|
||||
elif issue == "LOCK_EXPIRED":
|
||||
self.gov.revoke_agent(agent_id, RevocationType.LOCK_EXPIRED,
|
||||
"Lock expired while agent was running")
|
||||
return "REVOKED:LOCK"
|
||||
|
||||
elif issue.startswith("STUCK_"):
|
||||
self.gov.revoke_agent(agent_id, RevocationType.HEARTBEAT_TIMEOUT,
|
||||
f"Agent stuck with no progress: {issue}")
|
||||
return f"REVOKED:{issue}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class ComplianceWatcher:
|
||||
"""
|
||||
Monitors agent compliance with procedures.
|
||||
- Checks required artifacts exist
|
||||
- Detects forbidden actions
|
||||
- Validates phase transitions
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.gov = GovernanceManager()
|
||||
|
||||
def check_agent(self, agent_id: str) -> dict:
|
||||
"""Check an agent's compliance"""
|
||||
result = {
|
||||
"agent_id": agent_id,
|
||||
"compliant": True,
|
||||
"violations": []
|
||||
}
|
||||
|
||||
state = self.gov.get_agent_state(agent_id)
|
||||
packet = self.gov.get_instruction_packet(agent_id)
|
||||
|
||||
if not state or not packet:
|
||||
result["compliant"] = False
|
||||
result["violations"].append("MISSING_STATE_OR_PACKET")
|
||||
return result
|
||||
|
||||
# Check if EXECUTE was entered without PLAN artifact
|
||||
if state.phase in [AgentPhase.EXECUTE, AgentPhase.VERIFY, AgentPhase.PACKAGE]:
|
||||
if not self.gov.has_required_artifact(packet.task_id, "plan"):
|
||||
result["compliant"] = False
|
||||
result["violations"].append("EXECUTE_WITHOUT_PLAN_ARTIFACT")
|
||||
|
||||
# Check required steps from constraints
|
||||
required_steps = packet.constraints.get("required_steps", [])
|
||||
# This would need more sophisticated tracking in production
|
||||
|
||||
return result
|
||||
|
||||
def enforce(self, agent_id: str) -> Optional[str]:
|
||||
"""Enforce compliance, return action taken"""
|
||||
check = self.check_agent(agent_id)
|
||||
|
||||
if check["compliant"]:
|
||||
return None
|
||||
|
||||
for violation in check["violations"]:
|
||||
if violation == "EXECUTE_WITHOUT_PLAN_ARTIFACT":
|
||||
self.gov.revoke_agent(agent_id, RevocationType.PROCEDURE_VIOLATION,
|
||||
"Attempted EXECUTE phase without plan artifact")
|
||||
return "REVOKED:NO_PLAN_ARTIFACT"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class VaultLeaseWatcher:
|
||||
"""
|
||||
Monitors Vault token/lease validity.
|
||||
- Checks token accessibility
|
||||
- Confirms revocation signals
|
||||
- Enforces token revocation
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.gov = GovernanceManager()
|
||||
self.db = self.gov.db
|
||||
|
||||
def set_revocation_signal(self, agent_id: str):
|
||||
"""Set a signal that this agent should be revoked"""
|
||||
self.db.set(f"agent:{agent_id}:revoke_signal", "1", ex=300)
|
||||
|
||||
def has_revocation_signal(self, agent_id: str) -> bool:
|
||||
"""Check if revocation signal is set"""
|
||||
return self.db.exists(f"agent:{agent_id}:revoke_signal")
|
||||
|
||||
def clear_revocation_signal(self, agent_id: str):
|
||||
"""Clear revocation signal after enforcement"""
|
||||
self.db.delete(f"agent:{agent_id}:revoke_signal")
|
||||
|
||||
def enforce(self, agent_id: str) -> Optional[str]:
|
||||
"""Enforce revocation signal"""
|
||||
if self.has_revocation_signal(agent_id):
|
||||
self.gov.revoke_agent(agent_id, RevocationType.MANUAL,
|
||||
"Revocation signal received")
|
||||
self.clear_revocation_signal(agent_id)
|
||||
return "REVOKED:SIGNAL"
|
||||
return None
|
||||
|
||||
|
||||
class GovernanceMonitorDaemon:
|
||||
"""
|
||||
Background daemon that runs all monitors periodically.
|
||||
"""
|
||||
|
||||
def __init__(self, interval: int = 10):
|
||||
self.interval = interval
|
||||
self.gov = GovernanceManager()
|
||||
self.execution_watcher = ExecutionWatcher()
|
||||
self.compliance_watcher = ComplianceWatcher()
|
||||
self.vault_watcher = VaultLeaseWatcher()
|
||||
self.running = False
|
||||
|
||||
def get_active_agents(self) -> list[str]:
|
||||
"""Get list of agents that need monitoring"""
|
||||
# Get all agent state keys
|
||||
keys = self.gov.db.keys("agent:*:state")
|
||||
agents = []
|
||||
for key in keys:
|
||||
agent_id = key.split(":")[1]
|
||||
state = self.gov.get_agent_state(agent_id)
|
||||
if state and state.status == AgentStatus.RUNNING:
|
||||
agents.append(agent_id)
|
||||
return agents
|
||||
|
||||
def run_checks(self) -> list[dict]:
|
||||
"""Run all monitors once"""
|
||||
results = []
|
||||
|
||||
for agent_id in self.get_active_agents():
|
||||
result = {
|
||||
"agent_id": agent_id,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"actions": []
|
||||
}
|
||||
|
||||
# Execution check
|
||||
action = self.execution_watcher.enforce(agent_id)
|
||||
if action:
|
||||
result["actions"].append(action)
|
||||
|
||||
# Compliance check (only if not already revoked)
|
||||
state = self.gov.get_agent_state(agent_id)
|
||||
if state and state.status != AgentStatus.REVOKED:
|
||||
action = self.compliance_watcher.enforce(agent_id)
|
||||
if action:
|
||||
result["actions"].append(action)
|
||||
|
||||
# Vault check
|
||||
if state and state.status != AgentStatus.REVOKED:
|
||||
action = self.vault_watcher.enforce(agent_id)
|
||||
if action:
|
||||
result["actions"].append(action)
|
||||
|
||||
if result["actions"]:
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
def run_daemon(self):
|
||||
"""Run as a daemon (blocking)"""
|
||||
self.running = True
|
||||
print(f"[MONITOR] Starting governance monitor daemon (interval: {self.interval}s)")
|
||||
|
||||
while self.running:
|
||||
results = self.run_checks()
|
||||
for r in results:
|
||||
print(f"[MONITOR] {r['agent_id']}: {r['actions']}")
|
||||
time.sleep(self.interval)
|
||||
|
||||
def stop(self):
|
||||
"""Stop the daemon"""
|
||||
self.running = False
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: monitors.py <command>")
|
||||
print("Commands:")
|
||||
print(" check <agent_id> - Run all checks on an agent")
|
||||
print(" daemon - Run monitor daemon")
|
||||
print(" signal <agent_id> - Send revocation signal")
|
||||
sys.exit(1)
|
||||
|
||||
cmd = sys.argv[1]
|
||||
|
||||
if cmd == "check":
|
||||
agent_id = sys.argv[2]
|
||||
|
||||
exec_watch = ExecutionWatcher()
|
||||
comp_watch = ComplianceWatcher()
|
||||
|
||||
print(f"=== Checking Agent: {agent_id} ===\n")
|
||||
|
||||
print("Execution Check:")
|
||||
result = exec_watch.check_agent(agent_id)
|
||||
print(f" Healthy: {result['healthy']}")
|
||||
print(f" Issues: {result['issues']}")
|
||||
|
||||
print("\nCompliance Check:")
|
||||
result = comp_watch.check_agent(agent_id)
|
||||
print(f" Compliant: {result['compliant']}")
|
||||
print(f" Violations: {result['violations']}")
|
||||
|
||||
elif cmd == "daemon":
|
||||
daemon = GovernanceMonitorDaemon(interval=10)
|
||||
try:
|
||||
daemon.run_daemon()
|
||||
except KeyboardInterrupt:
|
||||
daemon.stop()
|
||||
print("\n[MONITOR] Stopped")
|
||||
|
||||
elif cmd == "signal":
|
||||
agent_id = sys.argv[2]
|
||||
watcher = VaultLeaseWatcher()
|
||||
watcher.set_revocation_signal(agent_id)
|
||||
print(f"Revocation signal set for {agent_id}")
|
||||
|
||||
else:
|
||||
print(f"Unknown command: {cmd}")
|
||||
Executable
+527
@@ -0,0 +1,527 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Promotion Engine
|
||||
================
|
||||
Manages agent tier progression based on compliance metrics.
|
||||
Part of Phase 4: Promotion and Revocation Engine.
|
||||
|
||||
Promotion Rules:
|
||||
- T0 → T1: 5 compliant plan-only tasks, correct scoping, clear rollbacks
|
||||
- T1 → T2: N consecutive compliant sandbox runs, 2+ successful modifications
|
||||
- T2 → T3: Demonstrated complexity reduction, correct escalation behavior
|
||||
- T3 → T4: Rare, manual review required
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Configuration
|
||||
# =============================================================================
|
||||
|
||||
LEDGER_DB = "/opt/agent-governance/ledger/governance.db"
|
||||
|
||||
# Promotion requirements per tier
|
||||
PROMOTION_REQUIREMENTS = {
|
||||
0: { # T0 → T1
|
||||
"min_compliant_runs": 5,
|
||||
"min_consecutive_compliant": 3,
|
||||
"required_action_types": ["generate_plan"],
|
||||
"max_violations_30d": 0,
|
||||
"description": "5 compliant plan-only tasks, 3 consecutive, no violations"
|
||||
},
|
||||
1: { # T1 → T2
|
||||
"min_compliant_runs": 10,
|
||||
"min_consecutive_compliant": 5,
|
||||
"required_action_types": ["ansible_check", "terraform_plan"],
|
||||
"min_successful_modifications": 2,
|
||||
"max_violations_30d": 0,
|
||||
"description": "10 compliant runs, 5 consecutive, 2+ mods, no violations"
|
||||
},
|
||||
2: { # T2 → T3
|
||||
"min_compliant_runs": 25,
|
||||
"min_consecutive_compliant": 10,
|
||||
"required_action_types": ["ansible_run", "terraform_apply"],
|
||||
"min_successful_modifications": 5,
|
||||
"max_violations_30d": 0,
|
||||
"requires_manual_review": True,
|
||||
"description": "25 compliant runs, demonstrated complexity handling"
|
||||
},
|
||||
3: { # T3 → T4
|
||||
"min_compliant_runs": 50,
|
||||
"min_consecutive_compliant": 25,
|
||||
"requires_manual_review": True,
|
||||
"requires_admin_approval": True,
|
||||
"description": "Rare, requires admin approval and extensive track record"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class PromotionStatus(str, Enum):
|
||||
ELIGIBLE = "ELIGIBLE"
|
||||
NOT_ELIGIBLE = "NOT_ELIGIBLE"
|
||||
PENDING_REVIEW = "PENDING_REVIEW"
|
||||
APPROVED = "APPROVED"
|
||||
DENIED = "DENIED"
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromotionEvaluation:
|
||||
agent_id: str
|
||||
current_tier: int
|
||||
target_tier: int
|
||||
status: PromotionStatus
|
||||
requirements_met: dict
|
||||
requirements_missing: list
|
||||
recommendation: str
|
||||
evidence: dict
|
||||
timestamp: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"agent_id": self.agent_id,
|
||||
"current_tier": self.current_tier,
|
||||
"target_tier": self.target_tier,
|
||||
"status": self.status.value,
|
||||
"requirements_met": self.requirements_met,
|
||||
"requirements_missing": self.requirements_missing,
|
||||
"recommendation": self.recommendation,
|
||||
"evidence": self.evidence,
|
||||
"timestamp": self.timestamp
|
||||
}
|
||||
|
||||
|
||||
class PromotionEngine:
|
||||
"""
|
||||
Evaluates and processes agent tier promotions.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str = LEDGER_DB):
|
||||
self.db_path = db_path
|
||||
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def get_agent_metrics(self, agent_id: str) -> Optional[dict]:
|
||||
"""Get current metrics for an agent"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT * FROM agent_metrics WHERE agent_id = ?
|
||||
""", (agent_id,))
|
||||
|
||||
row = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if row:
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
def get_agent_history(self, agent_id: str, days: int = 30) -> dict:
|
||||
"""Get agent's action history for evaluation"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get total runs and successes
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
COUNT(*) as total_actions,
|
||||
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful_actions,
|
||||
SUM(CASE WHEN decision = 'EXECUTE' AND success = 1 THEN 1 ELSE 0 END) as successful_executions
|
||||
FROM agent_actions
|
||||
WHERE agent_id = ?
|
||||
AND datetime(timestamp) >= datetime('now', ?)
|
||||
""", (agent_id, f'-{days} days'))
|
||||
|
||||
action_stats = dict(cursor.fetchone())
|
||||
|
||||
# Get action types used
|
||||
cursor.execute("""
|
||||
SELECT DISTINCT action FROM agent_actions
|
||||
WHERE agent_id = ?
|
||||
AND datetime(timestamp) >= datetime('now', ?)
|
||||
""", (agent_id, f'-{days} days'))
|
||||
|
||||
action_types = [row['action'] for row in cursor.fetchall()]
|
||||
|
||||
# Get violation count
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) as violation_count
|
||||
FROM violations
|
||||
WHERE agent_id = ?
|
||||
AND datetime(timestamp) >= datetime('now', ?)
|
||||
""", (agent_id, f'-{days} days'))
|
||||
|
||||
violations = cursor.fetchone()['violation_count']
|
||||
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
"total_actions": action_stats['total_actions'],
|
||||
"successful_actions": action_stats['successful_actions'],
|
||||
"successful_executions": action_stats['successful_executions'],
|
||||
"action_types": action_types,
|
||||
"violations_30d": violations
|
||||
}
|
||||
|
||||
def evaluate_promotion(self, agent_id: str) -> PromotionEvaluation:
|
||||
"""Evaluate if an agent is eligible for promotion"""
|
||||
metrics = self.get_agent_metrics(agent_id)
|
||||
|
||||
if not metrics:
|
||||
return PromotionEvaluation(
|
||||
agent_id=agent_id,
|
||||
current_tier=0,
|
||||
target_tier=1,
|
||||
status=PromotionStatus.NOT_ELIGIBLE,
|
||||
requirements_met={},
|
||||
requirements_missing=["Agent not found in metrics"],
|
||||
recommendation="Agent has no recorded activity",
|
||||
evidence={},
|
||||
timestamp=self._now()
|
||||
)
|
||||
|
||||
current_tier = metrics['current_tier']
|
||||
target_tier = current_tier + 1
|
||||
|
||||
if target_tier > 4:
|
||||
return PromotionEvaluation(
|
||||
agent_id=agent_id,
|
||||
current_tier=current_tier,
|
||||
target_tier=current_tier,
|
||||
status=PromotionStatus.NOT_ELIGIBLE,
|
||||
requirements_met={},
|
||||
requirements_missing=["Already at maximum tier"],
|
||||
recommendation="Agent is at Tier 4 (maximum)",
|
||||
evidence={"current_tier": current_tier},
|
||||
timestamp=self._now()
|
||||
)
|
||||
|
||||
requirements = PROMOTION_REQUIREMENTS.get(current_tier, {})
|
||||
history = self.get_agent_history(agent_id)
|
||||
|
||||
# Evaluate each requirement
|
||||
met = {}
|
||||
missing = []
|
||||
|
||||
# Check compliant runs
|
||||
min_runs = requirements.get('min_compliant_runs', 0)
|
||||
if metrics['compliant_runs'] >= min_runs:
|
||||
met['compliant_runs'] = f"{metrics['compliant_runs']} >= {min_runs}"
|
||||
else:
|
||||
missing.append(f"Need {min_runs - metrics['compliant_runs']} more compliant runs")
|
||||
|
||||
# Check consecutive compliant
|
||||
min_consecutive = requirements.get('min_consecutive_compliant', 0)
|
||||
if metrics['consecutive_compliant'] >= min_consecutive:
|
||||
met['consecutive_compliant'] = f"{metrics['consecutive_compliant']} >= {min_consecutive}"
|
||||
else:
|
||||
missing.append(f"Need {min_consecutive - metrics['consecutive_compliant']} more consecutive compliant runs")
|
||||
|
||||
# Check violations
|
||||
max_violations = requirements.get('max_violations_30d', 0)
|
||||
if history['violations_30d'] <= max_violations:
|
||||
met['violations_30d'] = f"{history['violations_30d']} <= {max_violations}"
|
||||
else:
|
||||
missing.append(f"Too many violations ({history['violations_30d']} > {max_violations})")
|
||||
|
||||
# Check action types
|
||||
required_types = requirements.get('required_action_types', [])
|
||||
for action_type in required_types:
|
||||
if action_type in history['action_types']:
|
||||
met[f'action_{action_type}'] = f"Has performed {action_type}"
|
||||
else:
|
||||
missing.append(f"Needs to perform action type: {action_type}")
|
||||
|
||||
# Check successful modifications (for T1+)
|
||||
min_mods = requirements.get('min_successful_modifications', 0)
|
||||
if min_mods > 0:
|
||||
if history['successful_executions'] >= min_mods:
|
||||
met['modifications'] = f"{history['successful_executions']} >= {min_mods}"
|
||||
else:
|
||||
missing.append(f"Need {min_mods - history['successful_executions']} more successful modifications")
|
||||
|
||||
# Determine status
|
||||
if len(missing) == 0:
|
||||
if requirements.get('requires_manual_review', False):
|
||||
status = PromotionStatus.PENDING_REVIEW
|
||||
recommendation = "All automatic criteria met. Manual review required."
|
||||
else:
|
||||
status = PromotionStatus.ELIGIBLE
|
||||
recommendation = "Agent meets all promotion criteria"
|
||||
else:
|
||||
status = PromotionStatus.NOT_ELIGIBLE
|
||||
recommendation = f"Agent needs to complete {len(missing)} requirements"
|
||||
|
||||
return PromotionEvaluation(
|
||||
agent_id=agent_id,
|
||||
current_tier=current_tier,
|
||||
target_tier=target_tier,
|
||||
status=status,
|
||||
requirements_met=met,
|
||||
requirements_missing=missing,
|
||||
recommendation=recommendation,
|
||||
evidence={
|
||||
"metrics": metrics,
|
||||
"history": history,
|
||||
"requirements": requirements
|
||||
},
|
||||
timestamp=self._now()
|
||||
)
|
||||
|
||||
def process_promotion(self, agent_id: str, approved_by: str,
|
||||
rationale: str = None) -> tuple[bool, str]:
|
||||
"""Process an approved promotion"""
|
||||
evaluation = self.evaluate_promotion(agent_id)
|
||||
|
||||
if evaluation.status not in [PromotionStatus.ELIGIBLE, PromotionStatus.PENDING_REVIEW]:
|
||||
return False, f"Agent not eligible: {evaluation.recommendation}"
|
||||
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
# Update agent tier in metrics
|
||||
cursor.execute("""
|
||||
UPDATE agent_metrics
|
||||
SET current_tier = ?,
|
||||
updated_at = ?
|
||||
WHERE agent_id = ?
|
||||
""", (evaluation.target_tier, self._now(), agent_id))
|
||||
|
||||
# Record promotion
|
||||
cursor.execute("""
|
||||
INSERT INTO promotions (timestamp, agent_id, from_tier, to_tier,
|
||||
approved_by, rationale, evidence)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
self._now(),
|
||||
agent_id,
|
||||
evaluation.current_tier,
|
||||
evaluation.target_tier,
|
||||
approved_by,
|
||||
rationale or evaluation.recommendation,
|
||||
json.dumps(evaluation.evidence)
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Update Vault policy
|
||||
self._update_vault_policy(agent_id, evaluation.target_tier)
|
||||
|
||||
return True, f"Promoted {agent_id} from Tier {evaluation.current_tier} to Tier {evaluation.target_tier}"
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
return False, f"Promotion failed: {str(e)}"
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _update_vault_policy(self, agent_id: str, new_tier: int):
|
||||
"""Update agent's Vault policy to new tier"""
|
||||
# In production, this would update the AppRole's token_policies
|
||||
# For now, just log the action
|
||||
print(f"[VAULT] Would update {agent_id} to tier{new_tier}-agent role")
|
||||
|
||||
# Example Vault API call (commented out for safety):
|
||||
# curl -sk -X POST \
|
||||
# -H "X-Vault-Token: $TOKEN" \
|
||||
# -d '{"token_policies": ["t{new_tier}-observer"]}' \
|
||||
# https://127.0.0.1:8200/v1/auth/approle/role/tier{new_tier}-agent
|
||||
|
||||
def get_promotion_history(self, agent_id: str = None, limit: int = 20) -> list:
|
||||
"""Get promotion history"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
if agent_id:
|
||||
cursor.execute("""
|
||||
SELECT * FROM promotions
|
||||
WHERE agent_id = ?
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ?
|
||||
""", (agent_id, limit))
|
||||
else:
|
||||
cursor.execute("""
|
||||
SELECT * FROM promotions
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ?
|
||||
""", (limit,))
|
||||
|
||||
rows = [dict(row) for row in cursor.fetchall()]
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
def get_all_eligible_agents(self) -> list[PromotionEvaluation]:
|
||||
"""Find all agents eligible for promotion"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT agent_id FROM agent_metrics")
|
||||
agents = [row['agent_id'] for row in cursor.fetchall()]
|
||||
conn.close()
|
||||
|
||||
eligible = []
|
||||
for agent_id in agents:
|
||||
eval = self.evaluate_promotion(agent_id)
|
||||
if eval.status in [PromotionStatus.ELIGIBLE, PromotionStatus.PENDING_REVIEW]:
|
||||
eligible.append(eval)
|
||||
|
||||
return eligible
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Promotion Engine")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
# Evaluate command
|
||||
eval_parser = subparsers.add_parser("evaluate", help="Evaluate agent for promotion")
|
||||
eval_parser.add_argument("agent_id", help="Agent ID to evaluate")
|
||||
eval_parser.add_argument("--json", action="store_true", help="Output JSON")
|
||||
|
||||
# Promote command
|
||||
promote_parser = subparsers.add_parser("promote", help="Process promotion")
|
||||
promote_parser.add_argument("agent_id", help="Agent ID to promote")
|
||||
promote_parser.add_argument("--approved-by", required=True, help="Approver name")
|
||||
promote_parser.add_argument("--rationale", help="Promotion rationale")
|
||||
|
||||
# List eligible command
|
||||
eligible_parser = subparsers.add_parser("eligible", help="List eligible agents")
|
||||
eligible_parser.add_argument("--json", action="store_true", help="Output JSON")
|
||||
|
||||
# History command
|
||||
history_parser = subparsers.add_parser("history", help="View promotion history")
|
||||
history_parser.add_argument("--agent-id", help="Filter by agent")
|
||||
history_parser.add_argument("--limit", type=int, default=20)
|
||||
history_parser.add_argument("--json", action="store_true", help="Output JSON")
|
||||
|
||||
# Requirements command
|
||||
req_parser = subparsers.add_parser("requirements", help="Show promotion requirements")
|
||||
req_parser.add_argument("--tier", type=int, help="Specific tier")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
engine = PromotionEngine()
|
||||
|
||||
if args.command == "evaluate":
|
||||
result = engine.evaluate_promotion(args.agent_id)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result.to_dict(), indent=2))
|
||||
else:
|
||||
print("\n" + "=" * 60)
|
||||
print("PROMOTION EVALUATION")
|
||||
print("=" * 60)
|
||||
print(f"Agent: {result.agent_id}")
|
||||
print(f"Current Tier: {result.current_tier}")
|
||||
print(f"Target Tier: {result.target_tier}")
|
||||
print(f"Status: {result.status.value}")
|
||||
print()
|
||||
|
||||
print("Requirements Met:")
|
||||
for key, value in result.requirements_met.items():
|
||||
print(f" [OK] {key}: {value}")
|
||||
|
||||
if result.requirements_missing:
|
||||
print("\nRequirements Missing:")
|
||||
for req in result.requirements_missing:
|
||||
print(f" [X] {req}")
|
||||
|
||||
print(f"\nRecommendation: {result.recommendation}")
|
||||
print("=" * 60)
|
||||
|
||||
elif args.command == "promote":
|
||||
success, message = engine.process_promotion(
|
||||
args.agent_id,
|
||||
args.approved_by,
|
||||
args.rationale
|
||||
)
|
||||
|
||||
if success:
|
||||
print(f"[OK] {message}")
|
||||
else:
|
||||
print(f"[FAILED] {message}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == "eligible":
|
||||
eligible = engine.get_all_eligible_agents()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([e.to_dict() for e in eligible], indent=2))
|
||||
else:
|
||||
print("\n" + "=" * 60)
|
||||
print("AGENTS ELIGIBLE FOR PROMOTION")
|
||||
print("=" * 60)
|
||||
|
||||
if not eligible:
|
||||
print("No agents currently eligible for promotion")
|
||||
else:
|
||||
for e in eligible:
|
||||
status_icon = "[READY]" if e.status == PromotionStatus.ELIGIBLE else "[REVIEW]"
|
||||
print(f"\n{status_icon} {e.agent_id}")
|
||||
print(f" Tier {e.current_tier} → Tier {e.target_tier}")
|
||||
print(f" {e.recommendation}")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
elif args.command == "history":
|
||||
history = engine.get_promotion_history(args.agent_id, args.limit)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(history, indent=2))
|
||||
else:
|
||||
print("\n" + "=" * 60)
|
||||
print("PROMOTION HISTORY")
|
||||
print("=" * 60)
|
||||
|
||||
if not history:
|
||||
print("No promotion history found")
|
||||
else:
|
||||
for p in history:
|
||||
print(f"\n{p['timestamp']}")
|
||||
print(f" {p['agent_id']}: Tier {p['from_tier']} → Tier {p['to_tier']}")
|
||||
print(f" Approved by: {p['approved_by']}")
|
||||
if p['rationale']:
|
||||
print(f" Rationale: {p['rationale']}")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
elif args.command == "requirements":
|
||||
print("\n" + "=" * 60)
|
||||
print("PROMOTION REQUIREMENTS")
|
||||
print("=" * 60)
|
||||
|
||||
tiers = [args.tier] if args.tier is not None else range(4)
|
||||
|
||||
for tier in tiers:
|
||||
req = PROMOTION_REQUIREMENTS.get(tier, {})
|
||||
print(f"\nTier {tier} → Tier {tier + 1}:")
|
||||
print(f" {req.get('description', 'No description')}")
|
||||
print(f" - Min compliant runs: {req.get('min_compliant_runs', 'N/A')}")
|
||||
print(f" - Min consecutive: {req.get('min_consecutive_compliant', 'N/A')}")
|
||||
print(f" - Max violations (30d): {req.get('max_violations_30d', 'N/A')}")
|
||||
if req.get('requires_manual_review'):
|
||||
print(f" - Requires manual review: Yes")
|
||||
if req.get('requires_admin_approval'):
|
||||
print(f" - Requires admin approval: Yes")
|
||||
|
||||
print("=" * 60)
|
||||
Executable
+637
@@ -0,0 +1,637 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Revocation Engine
|
||||
=================
|
||||
Real-time violation detection and immediate credential revocation.
|
||||
Part of Phase 4: Promotion and Revocation Engine.
|
||||
|
||||
Immediate Revocation Events:
|
||||
- Resource created outside approved pool
|
||||
- Terraform apply without stored plan
|
||||
- Ansible run without check-mode (no waiver)
|
||||
- Prod access without gate approval
|
||||
- Unrecorded root session
|
||||
- Direct baseline mutation
|
||||
- Error budget exceeded
|
||||
- Procedure violations
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import redis
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Configuration
|
||||
# =============================================================================
|
||||
|
||||
LEDGER_DB = "/opt/agent-governance/ledger/governance.db"
|
||||
|
||||
|
||||
class ViolationType(str, Enum):
|
||||
# Critical - Immediate revocation
|
||||
UNAUTHORIZED_POOL = "UNAUTHORIZED_POOL"
|
||||
APPLY_WITHOUT_PLAN = "APPLY_WITHOUT_PLAN"
|
||||
RUN_WITHOUT_CHECK = "RUN_WITHOUT_CHECK"
|
||||
UNAUTHORIZED_PROD = "UNAUTHORIZED_PROD"
|
||||
UNRECORDED_ROOT = "UNRECORDED_ROOT"
|
||||
BASELINE_MUTATION = "BASELINE_MUTATION"
|
||||
|
||||
# High - Immediate revocation
|
||||
ERROR_BUDGET_EXCEEDED = "ERROR_BUDGET_EXCEEDED"
|
||||
PROCEDURE_VIOLATION = "PROCEDURE_VIOLATION"
|
||||
HEARTBEAT_TIMEOUT = "HEARTBEAT_TIMEOUT"
|
||||
LOCK_EXPIRED = "LOCK_EXPIRED"
|
||||
|
||||
# Medium - Warning then revocation
|
||||
SCOPE_VIOLATION = "SCOPE_VIOLATION"
|
||||
FORBIDDEN_ACTION = "FORBIDDEN_ACTION"
|
||||
|
||||
# Low - Warning only
|
||||
CONFIDENCE_BELOW_THRESHOLD = "CONFIDENCE_BELOW_THRESHOLD"
|
||||
MISSING_ARTIFACT = "MISSING_ARTIFACT"
|
||||
|
||||
|
||||
class Severity(str, Enum):
|
||||
CRITICAL = "critical" # Immediate revocation, alert
|
||||
HIGH = "high" # Immediate revocation
|
||||
MEDIUM = "medium" # Warning, second offense = revoke
|
||||
LOW = "low" # Warning only
|
||||
|
||||
|
||||
# Violation severity mapping
|
||||
VIOLATION_SEVERITY = {
|
||||
ViolationType.UNAUTHORIZED_POOL: Severity.CRITICAL,
|
||||
ViolationType.APPLY_WITHOUT_PLAN: Severity.CRITICAL,
|
||||
ViolationType.RUN_WITHOUT_CHECK: Severity.CRITICAL,
|
||||
ViolationType.UNAUTHORIZED_PROD: Severity.CRITICAL,
|
||||
ViolationType.UNRECORDED_ROOT: Severity.CRITICAL,
|
||||
ViolationType.BASELINE_MUTATION: Severity.CRITICAL,
|
||||
ViolationType.ERROR_BUDGET_EXCEEDED: Severity.HIGH,
|
||||
ViolationType.PROCEDURE_VIOLATION: Severity.HIGH,
|
||||
ViolationType.HEARTBEAT_TIMEOUT: Severity.HIGH,
|
||||
ViolationType.LOCK_EXPIRED: Severity.HIGH,
|
||||
ViolationType.SCOPE_VIOLATION: Severity.MEDIUM,
|
||||
ViolationType.FORBIDDEN_ACTION: Severity.MEDIUM,
|
||||
ViolationType.CONFIDENCE_BELOW_THRESHOLD: Severity.LOW,
|
||||
ViolationType.MISSING_ARTIFACT: Severity.LOW,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Violation:
|
||||
agent_id: str
|
||||
violation_type: ViolationType
|
||||
severity: Severity
|
||||
description: str
|
||||
triggering_action: str
|
||||
evidence: dict
|
||||
timestamp: str
|
||||
remediation: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"agent_id": self.agent_id,
|
||||
"violation_type": self.violation_type.value,
|
||||
"severity": self.severity.value,
|
||||
"description": self.description,
|
||||
"triggering_action": self.triggering_action,
|
||||
"evidence": self.evidence,
|
||||
"timestamp": self.timestamp,
|
||||
"remediation": self.remediation
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RevocationResult:
|
||||
agent_id: str
|
||||
success: bool
|
||||
action_taken: str
|
||||
violation: Violation
|
||||
vault_revoked: bool
|
||||
dragonfly_revoked: bool
|
||||
timestamp: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"agent_id": self.agent_id,
|
||||
"success": self.success,
|
||||
"action_taken": self.action_taken,
|
||||
"violation": self.violation.to_dict(),
|
||||
"vault_revoked": self.vault_revoked,
|
||||
"dragonfly_revoked": self.dragonfly_revoked,
|
||||
"timestamp": self.timestamp
|
||||
}
|
||||
|
||||
|
||||
class RevocationEngine:
|
||||
"""
|
||||
Detects violations and revokes agent access.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str = LEDGER_DB):
|
||||
self.db_path = db_path
|
||||
self.vault_token = self._get_vault_token()
|
||||
self.redis = self._get_redis()
|
||||
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _get_vault_token(self) -> str:
|
||||
try:
|
||||
with open("/opt/vault/init-keys.json") as f:
|
||||
return json.load(f)["root_token"]
|
||||
except:
|
||||
return ""
|
||||
|
||||
def _get_redis(self) -> Optional[redis.Redis]:
|
||||
try:
|
||||
# Get credentials from Vault
|
||||
result = subprocess.run([
|
||||
"curl", "-sk",
|
||||
"-H", f"X-Vault-Token: {self.vault_token}",
|
||||
"https://127.0.0.1:8200/v1/secret/data/services/dragonfly"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
creds = json.loads(result.stdout)["data"]["data"]
|
||||
return redis.Redis(
|
||||
host=creds["host"],
|
||||
port=int(creds["port"]),
|
||||
password=creds["password"],
|
||||
decode_responses=True
|
||||
)
|
||||
except:
|
||||
return None
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def record_violation(self, violation: Violation) -> int:
|
||||
"""Record a violation in the ledger"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
INSERT INTO violations
|
||||
(timestamp, agent_id, violation_type, severity, description,
|
||||
triggering_action, evidence, remediation)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
violation.timestamp,
|
||||
violation.agent_id,
|
||||
violation.violation_type.value,
|
||||
violation.severity.value,
|
||||
violation.description,
|
||||
violation.triggering_action,
|
||||
json.dumps(violation.evidence),
|
||||
violation.remediation
|
||||
))
|
||||
|
||||
violation_id = cursor.lastrowid
|
||||
|
||||
# Update agent metrics
|
||||
cursor.execute("""
|
||||
UPDATE agent_metrics
|
||||
SET consecutive_compliant = 0,
|
||||
last_violation_at = ?,
|
||||
promotion_eligible = 0,
|
||||
updated_at = ?
|
||||
WHERE agent_id = ?
|
||||
""", (violation.timestamp, violation.timestamp, violation.agent_id))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return violation_id
|
||||
|
||||
def revoke_vault_token(self, agent_id: str) -> bool:
|
||||
"""Revoke agent's Vault token"""
|
||||
# In production, this would look up the agent's token accessor
|
||||
# and revoke it via Vault API
|
||||
|
||||
# For now, set a revocation signal that the agent must respect
|
||||
try:
|
||||
result = subprocess.run([
|
||||
"curl", "-sk", "-X", "POST",
|
||||
"-H", f"X-Vault-Token: {self.vault_token}",
|
||||
"-d", json.dumps({"revoked_at": self._now(), "agent_id": agent_id}),
|
||||
f"https://127.0.0.1:8200/v1/secret/data/revocations/{agent_id}"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
return "errors" not in result.stdout
|
||||
except:
|
||||
return False
|
||||
|
||||
def revoke_dragonfly_access(self, agent_id: str) -> bool:
|
||||
"""Revoke agent's access in DragonflyDB"""
|
||||
if not self.redis:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Set revocation signal
|
||||
self.redis.set(f"agent:{agent_id}:revoke_signal", "1", ex=86400)
|
||||
|
||||
# Update agent state to REVOKED
|
||||
state_key = f"agent:{agent_id}:state"
|
||||
state_data = self.redis.get(state_key)
|
||||
if state_data:
|
||||
state = json.loads(state_data)
|
||||
state["status"] = "REVOKED"
|
||||
state["revoked_at"] = self._now()
|
||||
self.redis.set(state_key, json.dumps(state))
|
||||
|
||||
# Release any locks
|
||||
self.redis.delete(f"agent:{agent_id}:lock")
|
||||
|
||||
# Add to revocation ledger
|
||||
revocation_event = {
|
||||
"agent_id": agent_id,
|
||||
"revoked_at": self._now(),
|
||||
"reason": "VIOLATION"
|
||||
}
|
||||
self.redis.rpush("revocations:ledger", json.dumps(revocation_event))
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Failed to revoke DragonflyDB access: {e}")
|
||||
return False
|
||||
|
||||
def send_alert(self, violation: Violation):
|
||||
"""Send alert for critical violations"""
|
||||
# In production, this would integrate with PagerDuty, Slack, etc.
|
||||
print(f"\n{'!'*60}")
|
||||
print(f"CRITICAL ALERT: {violation.violation_type.value}")
|
||||
print(f"{'!'*60}")
|
||||
print(f"Agent: {violation.agent_id}")
|
||||
print(f"Description: {violation.description}")
|
||||
print(f"Severity: {violation.severity.value}")
|
||||
print(f"Time: {violation.timestamp}")
|
||||
print(f"{'!'*60}\n")
|
||||
|
||||
# Store alert for dashboard
|
||||
if self.redis:
|
||||
alert = {
|
||||
"type": "VIOLATION",
|
||||
"violation_type": violation.violation_type.value,
|
||||
"agent_id": violation.agent_id,
|
||||
"severity": violation.severity.value,
|
||||
"message": violation.description,
|
||||
"timestamp": violation.timestamp
|
||||
}
|
||||
self.redis.rpush("alerts:queue", json.dumps(alert))
|
||||
self.redis.ltrim("alerts:queue", -100, -1) # Keep last 100 alerts
|
||||
|
||||
def process_violation(self, violation: Violation) -> RevocationResult:
|
||||
"""Process a violation and take appropriate action"""
|
||||
print(f"\n[VIOLATION] Processing: {violation.violation_type.value}")
|
||||
print(f"[VIOLATION] Agent: {violation.agent_id}")
|
||||
print(f"[VIOLATION] Severity: {violation.severity.value}")
|
||||
|
||||
# Record the violation
|
||||
violation_id = self.record_violation(violation)
|
||||
|
||||
# Determine action based on severity
|
||||
vault_revoked = False
|
||||
dragonfly_revoked = False
|
||||
action_taken = "RECORDED"
|
||||
|
||||
if violation.severity in [Severity.CRITICAL, Severity.HIGH]:
|
||||
# Immediate revocation
|
||||
print(f"[REVOKE] Initiating immediate revocation for {violation.agent_id}")
|
||||
|
||||
vault_revoked = self.revoke_vault_token(violation.agent_id)
|
||||
dragonfly_revoked = self.revoke_dragonfly_access(violation.agent_id)
|
||||
|
||||
action_taken = "REVOKED"
|
||||
|
||||
if violation.severity == Severity.CRITICAL:
|
||||
self.send_alert(violation)
|
||||
|
||||
elif violation.severity == Severity.MEDIUM:
|
||||
# Check for prior warnings
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) as count FROM violations
|
||||
WHERE agent_id = ?
|
||||
AND severity = 'medium'
|
||||
AND datetime(timestamp) >= datetime('now', '-7 days')
|
||||
""", (violation.agent_id,))
|
||||
|
||||
prior_warnings = cursor.fetchone()['count']
|
||||
conn.close()
|
||||
|
||||
if prior_warnings >= 2: # This is the 3rd+ medium violation
|
||||
print(f"[REVOKE] Multiple medium violations, revoking {violation.agent_id}")
|
||||
vault_revoked = self.revoke_vault_token(violation.agent_id)
|
||||
dragonfly_revoked = self.revoke_dragonfly_access(violation.agent_id)
|
||||
action_taken = "REVOKED"
|
||||
else:
|
||||
print(f"[WARNING] Warning issued to {violation.agent_id}")
|
||||
action_taken = "WARNING"
|
||||
|
||||
else: # LOW severity
|
||||
print(f"[WARNING] Low severity violation recorded for {violation.agent_id}")
|
||||
action_taken = "WARNING"
|
||||
|
||||
return RevocationResult(
|
||||
agent_id=violation.agent_id,
|
||||
success=True,
|
||||
action_taken=action_taken,
|
||||
violation=violation,
|
||||
vault_revoked=vault_revoked,
|
||||
dragonfly_revoked=dragonfly_revoked,
|
||||
timestamp=self._now()
|
||||
)
|
||||
|
||||
def create_violation(
|
||||
self,
|
||||
agent_id: str,
|
||||
violation_type: ViolationType,
|
||||
description: str,
|
||||
triggering_action: str = "",
|
||||
evidence: dict = None,
|
||||
remediation: str = ""
|
||||
) -> Violation:
|
||||
"""Helper to create a violation object"""
|
||||
return Violation(
|
||||
agent_id=agent_id,
|
||||
violation_type=violation_type,
|
||||
severity=VIOLATION_SEVERITY[violation_type],
|
||||
description=description,
|
||||
triggering_action=triggering_action,
|
||||
evidence=evidence or {},
|
||||
timestamp=self._now(),
|
||||
remediation=remediation
|
||||
)
|
||||
|
||||
def get_violations(self, agent_id: str = None, severity: str = None,
|
||||
limit: int = 50) -> list:
|
||||
"""Get violation history"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = "SELECT * FROM violations WHERE 1=1"
|
||||
params = []
|
||||
|
||||
if agent_id:
|
||||
query += " AND agent_id = ?"
|
||||
params.append(agent_id)
|
||||
|
||||
if severity:
|
||||
query += " AND severity = ?"
|
||||
params.append(severity)
|
||||
|
||||
query += " ORDER BY timestamp DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
|
||||
cursor.execute(query, params)
|
||||
rows = [dict(row) for row in cursor.fetchall()]
|
||||
conn.close()
|
||||
|
||||
return rows
|
||||
|
||||
def get_active_revocations(self) -> list:
|
||||
"""Get currently revoked agents from DragonflyDB"""
|
||||
if not self.redis:
|
||||
return []
|
||||
|
||||
revoked = []
|
||||
keys = self.redis.keys("agent:*:state")
|
||||
|
||||
for key in keys:
|
||||
data = self.redis.get(key)
|
||||
if data:
|
||||
state = json.loads(data)
|
||||
if state.get("status") == "REVOKED":
|
||||
revoked.append({
|
||||
"agent_id": state.get("agent_id"),
|
||||
"revoked_at": state.get("revoked_at"),
|
||||
"notes": state.get("notes", "")
|
||||
})
|
||||
|
||||
return revoked
|
||||
|
||||
def acknowledge_violation(self, violation_id: int, acknowledged_by: str) -> bool:
|
||||
"""Acknowledge a violation (for remediation tracking)"""
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
UPDATE violations
|
||||
SET acknowledged = 1,
|
||||
acknowledged_by = ?
|
||||
WHERE id = ?
|
||||
""", (acknowledged_by, violation_id))
|
||||
|
||||
success = cursor.rowcount > 0
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return success
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Violation Detectors
|
||||
# =============================================================================
|
||||
|
||||
class ViolationDetector:
|
||||
"""
|
||||
Detects violations in real-time.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.engine = RevocationEngine()
|
||||
|
||||
def check_plan_artifact(self, agent_id: str, action: str, artifact_exists: bool) -> Optional[Violation]:
|
||||
"""Check if apply/run has a corresponding plan artifact"""
|
||||
if action in ["terraform_apply", "ansible_run"] and not artifact_exists:
|
||||
return self.engine.create_violation(
|
||||
agent_id=agent_id,
|
||||
violation_type=ViolationType.APPLY_WITHOUT_PLAN,
|
||||
description=f"Attempted {action} without required plan artifact",
|
||||
triggering_action=action,
|
||||
remediation="Always run plan/check before apply/run"
|
||||
)
|
||||
return None
|
||||
|
||||
def check_pool_authorization(self, agent_id: str, agent_tier: int,
|
||||
target_pool: str) -> Optional[Violation]:
|
||||
"""Check if agent is authorized for target pool"""
|
||||
forbidden_pools = {
|
||||
0: ["pve-sandbox", "pve-staging", "pve-prod"],
|
||||
1: ["pve-staging", "pve-prod"],
|
||||
2: ["pve-prod"],
|
||||
3: [],
|
||||
4: []
|
||||
}
|
||||
|
||||
if target_pool in forbidden_pools.get(agent_tier, []):
|
||||
return self.engine.create_violation(
|
||||
agent_id=agent_id,
|
||||
violation_type=ViolationType.UNAUTHORIZED_POOL,
|
||||
description=f"Tier {agent_tier} agent accessed forbidden pool: {target_pool}",
|
||||
evidence={"agent_tier": agent_tier, "target_pool": target_pool},
|
||||
remediation="Request promotion or use authorized pools only"
|
||||
)
|
||||
return None
|
||||
|
||||
def check_production_access(self, agent_id: str, target: str,
|
||||
has_approval: bool) -> Optional[Violation]:
|
||||
"""Check if production access has gate approval"""
|
||||
if "prod" in target.lower() and not has_approval:
|
||||
return self.engine.create_violation(
|
||||
agent_id=agent_id,
|
||||
violation_type=ViolationType.UNAUTHORIZED_PROD,
|
||||
description=f"Production access attempted without approval: {target}",
|
||||
evidence={"target": target, "approval": False},
|
||||
remediation="Request production access through gate approval process"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Revocation Engine")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
# Report violation command
|
||||
report_parser = subparsers.add_parser("report", help="Report a violation")
|
||||
report_parser.add_argument("agent_id", help="Agent ID")
|
||||
report_parser.add_argument("--type", required=True,
|
||||
choices=[v.value for v in ViolationType],
|
||||
help="Violation type")
|
||||
report_parser.add_argument("--description", required=True, help="Description")
|
||||
report_parser.add_argument("--action", default="", help="Triggering action")
|
||||
|
||||
# List violations command
|
||||
list_parser = subparsers.add_parser("list", help="List violations")
|
||||
list_parser.add_argument("--agent-id", help="Filter by agent")
|
||||
list_parser.add_argument("--severity", choices=["critical", "high", "medium", "low"])
|
||||
list_parser.add_argument("--limit", type=int, default=20)
|
||||
list_parser.add_argument("--json", action="store_true")
|
||||
|
||||
# List revoked command
|
||||
revoked_parser = subparsers.add_parser("revoked", help="List revoked agents")
|
||||
revoked_parser.add_argument("--json", action="store_true")
|
||||
|
||||
# Acknowledge command
|
||||
ack_parser = subparsers.add_parser("acknowledge", help="Acknowledge violation")
|
||||
ack_parser.add_argument("violation_id", type=int)
|
||||
ack_parser.add_argument("--by", required=True, help="Acknowledger name")
|
||||
|
||||
# Types command
|
||||
types_parser = subparsers.add_parser("types", help="List violation types")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
engine = RevocationEngine()
|
||||
|
||||
if args.command == "report":
|
||||
violation_type = ViolationType(args.type)
|
||||
violation = engine.create_violation(
|
||||
agent_id=args.agent_id,
|
||||
violation_type=violation_type,
|
||||
description=args.description,
|
||||
triggering_action=args.action
|
||||
)
|
||||
|
||||
result = engine.process_violation(violation)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print("VIOLATION PROCESSED")
|
||||
print(f"{'='*60}")
|
||||
print(f"Agent: {result.agent_id}")
|
||||
print(f"Type: {result.violation.violation_type.value}")
|
||||
print(f"Severity: {result.violation.severity.value}")
|
||||
print(f"Action Taken: {result.action_taken}")
|
||||
print(f"Vault Revoked: {result.vault_revoked}")
|
||||
print(f"DragonflyDB Revoked: {result.dragonfly_revoked}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
elif args.command == "list":
|
||||
violations = engine.get_violations(
|
||||
agent_id=args.agent_id,
|
||||
severity=args.severity,
|
||||
limit=args.limit
|
||||
)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(violations, indent=2))
|
||||
else:
|
||||
print(f"\n{'='*60}")
|
||||
print("VIOLATION HISTORY")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if not violations:
|
||||
print("No violations found")
|
||||
else:
|
||||
for v in violations:
|
||||
severity_icon = {
|
||||
"critical": "[CRIT]",
|
||||
"high": "[HIGH]",
|
||||
"medium": "[MED]",
|
||||
"low": "[LOW]"
|
||||
}.get(v['severity'], "[?]")
|
||||
|
||||
ack = "[ACK]" if v['acknowledged'] else ""
|
||||
print(f"\n{severity_icon} {v['violation_type']} {ack}")
|
||||
print(f" Agent: {v['agent_id']}")
|
||||
print(f" {v['description']}")
|
||||
print(f" Time: {v['timestamp']}")
|
||||
|
||||
print(f"{'='*60}")
|
||||
|
||||
elif args.command == "revoked":
|
||||
revoked = engine.get_active_revocations()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(revoked, indent=2))
|
||||
else:
|
||||
print(f"\n{'='*60}")
|
||||
print("CURRENTLY REVOKED AGENTS")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if not revoked:
|
||||
print("No agents currently revoked")
|
||||
else:
|
||||
for r in revoked:
|
||||
print(f"\n {r['agent_id']}")
|
||||
print(f" Revoked: {r['revoked_at']}")
|
||||
if r['notes']:
|
||||
print(f" Notes: {r['notes']}")
|
||||
|
||||
print(f"{'='*60}")
|
||||
|
||||
elif args.command == "acknowledge":
|
||||
success = engine.acknowledge_violation(args.violation_id, args.by)
|
||||
if success:
|
||||
print(f"[OK] Violation {args.violation_id} acknowledged by {args.by}")
|
||||
else:
|
||||
print(f"[FAILED] Could not acknowledge violation {args.violation_id}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == "types":
|
||||
print(f"\n{'='*60}")
|
||||
print("VIOLATION TYPES")
|
||||
print(f"{'='*60}")
|
||||
|
||||
for vtype in ViolationType:
|
||||
severity = VIOLATION_SEVERITY[vtype]
|
||||
print(f"\n {vtype.value}")
|
||||
print(f" Severity: {severity.value}")
|
||||
|
||||
print(f"{'='*60}")
|
||||
Reference in New Issue
Block a user