Initial commit: Agent Governance System Phase 8

Phase 8 Production Hardening with complete governance infrastructure:

- Vault integration with tiered policies (T0-T4)
- DragonflyDB state management
- SQLite audit ledger
- Pipeline DSL and templates
- Promotion/revocation engine
- Checkpoint system for session persistence
- Health manager and circuit breaker for fault tolerance
- GitHub/Slack integrations
- Architectural test pipeline with bug watcher, suggestion engine, council review
- Multi-agent chaos testing framework

Test Results:
- Governance tests: 68/68 passing
- E2E workflow: 16/16 passing
- Phase 2 Vault: 14/14 passing
- Integration tests: 27/27 passing

Coverage: 57.6% average across 12 phases

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
2026-01-23 22:07:06 -05:00
co-authored by Claude Opus 4.5
commit 77655c298c
338 changed files with 63491 additions and 0 deletions
+35
View File
@@ -0,0 +1,35 @@
# Runtime
> Runtime governance and agent lifecycle
## Overview
This directory contains runtime governance and agent lifecycle.
## Key Files
| File | Description |
|------|-------------|
| `promotion.py` | Python module |
| `governance.py` | Python module |
| `revocation.py` | Python module |
| `monitors.py` | Python module |
## Interfaces / APIs
*Document any APIs, CLI commands, or interfaces here.*
## Status
** Not Started**
See [STATUS.md](./STATUS.md) for detailed progress tracking.
## Architecture Reference
Part of the [Agent Governance System](/opt/agent-governance/docs/ARCHITECTURE.md).
Parent: [Project Root](/opt/agent-governance)
---
*Last updated: 2026-01-23 23:25:09 UTC*
+35
View File
@@ -0,0 +1,35 @@
# Status: Runtime
## Current Phase
**COMPLETE**
## Tasks
| Status | Task | Updated |
|--------|------|---------|
| ☐ | *No tasks defined* | - |
## Dependencies
*No external dependencies.*
## Issues / Blockers
*No current issues or blockers.*
## Activity Log
### 2026-01-23 23:25:30 UTC
- **Phase**: COMPLETE
- **Action**: Governance runtime unified with pipeline.core
- **Details**: Phase updated to complete
### 2026-01-23 23:25:09 UTC
- **Phase**: NOT STARTED
- **Action**: Initialized
- **Details**: Status tracking initialized for this directory.
---
*Last updated: 2026-01-23 23:25:30 UTC*
+465
View File
@@ -0,0 +1,465 @@
#!/usr/bin/env python3
"""
Circuit Breaker
===============
Implements circuit breaker pattern for graceful degradation when dependencies fail.
States:
- CLOSED: Normal operation, requests flow through
- OPEN: Circuit tripped, requests fail fast
- HALF_OPEN: Testing if dependency recovered
Part of Phase 8: Production Hardening.
"""
import json
import time
import threading
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from functools import wraps
from typing import Any, Callable, Optional, TypeVar, Generic
import redis
class CircuitState(str, Enum):
"""Circuit breaker states"""
CLOSED = "closed" # Normal operation
OPEN = "open" # Failing fast
HALF_OPEN = "half_open" # Testing recovery
class FailureType(str, Enum):
"""Types of failures tracked"""
TIMEOUT = "timeout"
CONNECTION_ERROR = "connection_error"
EXCEPTION = "exception"
THRESHOLD_EXCEEDED = "threshold_exceeded"
@dataclass
class CircuitStats:
"""Statistics for a circuit breaker"""
total_calls: int = 0
successful_calls: int = 0
failed_calls: int = 0
rejected_calls: int = 0
last_failure_time: Optional[str] = None
last_success_time: Optional[str] = None
consecutive_failures: int = 0
consecutive_successes: int = 0
def to_dict(self) -> dict:
return {
"total_calls": self.total_calls,
"successful_calls": self.successful_calls,
"failed_calls": self.failed_calls,
"rejected_calls": self.rejected_calls,
"last_failure_time": self.last_failure_time,
"last_success_time": self.last_success_time,
"consecutive_failures": self.consecutive_failures,
"consecutive_successes": self.consecutive_successes,
"success_rate": self.success_rate
}
@property
def success_rate(self) -> float:
if self.total_calls == 0:
return 1.0
return self.successful_calls / self.total_calls
@dataclass
class CircuitConfig:
"""Configuration for a circuit breaker"""
name: str
failure_threshold: int = 5 # Failures before opening
success_threshold: int = 3 # Successes to close from half-open
timeout_seconds: float = 30.0 # Time before trying again (open -> half-open)
call_timeout: float = 10.0 # Timeout for individual calls
half_open_max_calls: int = 3 # Max concurrent calls in half-open
class CircuitOpenError(Exception):
"""Raised when circuit is open and rejecting calls"""
def __init__(self, circuit_name: str, retry_after: float):
self.circuit_name = circuit_name
self.retry_after = retry_after
super().__init__(f"Circuit '{circuit_name}' is open. Retry after {retry_after:.1f}s")
T = TypeVar('T')
class CircuitBreaker:
"""
Circuit breaker implementation for a single dependency.
Usage:
breaker = CircuitBreaker(CircuitConfig(name="vault"))
@breaker
def call_vault():
# ... make vault request
pass
# Or manual:
with breaker:
# ... protected code
"""
def __init__(self, config: CircuitConfig):
self.config = config
self._state = CircuitState.CLOSED
self._stats = CircuitStats()
self._last_failure_time: Optional[float] = None
self._half_open_calls = 0
self._lock = threading.RLock()
@property
def state(self) -> CircuitState:
with self._lock:
# Check if we should transition from OPEN to HALF_OPEN
if self._state == CircuitState.OPEN:
if self._last_failure_time:
elapsed = time.time() - self._last_failure_time
if elapsed >= self.config.timeout_seconds:
self._state = CircuitState.HALF_OPEN
self._half_open_calls = 0
return self._state
@property
def stats(self) -> CircuitStats:
return self._stats
def _now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _record_success(self):
"""Record a successful call"""
with self._lock:
self._stats.total_calls += 1
self._stats.successful_calls += 1
self._stats.consecutive_successes += 1
self._stats.consecutive_failures = 0
self._stats.last_success_time = self._now()
if self._state == CircuitState.HALF_OPEN:
if self._stats.consecutive_successes >= self.config.success_threshold:
self._state = CircuitState.CLOSED
self._stats.consecutive_successes = 0
def _record_failure(self, failure_type: FailureType = FailureType.EXCEPTION):
"""Record a failed call"""
with self._lock:
self._stats.total_calls += 1
self._stats.failed_calls += 1
self._stats.consecutive_failures += 1
self._stats.consecutive_successes = 0
self._stats.last_failure_time = self._now()
self._last_failure_time = time.time()
if self._state == CircuitState.CLOSED:
if self._stats.consecutive_failures >= self.config.failure_threshold:
self._state = CircuitState.OPEN
elif self._state == CircuitState.HALF_OPEN:
# Any failure in half-open goes back to open
self._state = CircuitState.OPEN
def _record_rejection(self):
"""Record a rejected call (circuit open)"""
with self._lock:
self._stats.total_calls += 1
self._stats.rejected_calls += 1
def _can_execute(self) -> bool:
"""Check if a call can be executed"""
state = self.state # This may transition OPEN -> HALF_OPEN
if state == CircuitState.CLOSED:
return True
if state == CircuitState.OPEN:
return False
if state == CircuitState.HALF_OPEN:
with self._lock:
if self._half_open_calls < self.config.half_open_max_calls:
self._half_open_calls += 1
return True
return False
return False
def _get_retry_after(self) -> float:
"""Get seconds until circuit might close"""
if self._last_failure_time:
elapsed = time.time() - self._last_failure_time
remaining = self.config.timeout_seconds - elapsed
return max(0, remaining)
return 0
def execute(self, func: Callable[[], T]) -> T:
"""Execute a function with circuit breaker protection"""
if not self._can_execute():
self._record_rejection()
raise CircuitOpenError(self.config.name, self._get_retry_after())
try:
result = func()
self._record_success()
return result
except Exception as e:
self._record_failure()
raise
def __call__(self, func: Callable) -> Callable:
"""Decorator usage"""
@wraps(func)
def wrapper(*args, **kwargs):
return self.execute(lambda: func(*args, **kwargs))
return wrapper
def __enter__(self):
"""Context manager entry"""
if not self._can_execute():
self._record_rejection()
raise CircuitOpenError(self.config.name, self._get_retry_after())
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Context manager exit"""
if exc_type is None:
self._record_success()
else:
self._record_failure()
return False # Don't suppress exceptions
def reset(self):
"""Manually reset the circuit breaker"""
with self._lock:
self._state = CircuitState.CLOSED
self._stats = CircuitStats()
self._last_failure_time = None
self._half_open_calls = 0
def force_open(self):
"""Manually open the circuit"""
with self._lock:
self._state = CircuitState.OPEN
self._last_failure_time = time.time()
def to_dict(self) -> dict:
"""Get circuit breaker state as dict"""
return {
"name": self.config.name,
"state": self.state.value,
"stats": self._stats.to_dict(),
"config": {
"failure_threshold": self.config.failure_threshold,
"success_threshold": self.config.success_threshold,
"timeout_seconds": self.config.timeout_seconds
}
}
class CircuitBreakerRegistry:
"""
Registry for managing multiple circuit breakers.
Usage:
registry = CircuitBreakerRegistry()
registry.register("vault", CircuitConfig(name="vault", failure_threshold=3))
@registry.protect("vault")
def call_vault():
# ...
"""
def __init__(self, redis_client: Optional[redis.Redis] = None):
self._breakers: dict[str, CircuitBreaker] = {}
self._redis = redis_client
self._lock = threading.RLock()
def register(self, name: str, config: Optional[CircuitConfig] = None) -> CircuitBreaker:
"""Register a circuit breaker"""
with self._lock:
if name not in self._breakers:
cfg = config or CircuitConfig(name=name)
self._breakers[name] = CircuitBreaker(cfg)
return self._breakers[name]
def get(self, name: str) -> Optional[CircuitBreaker]:
"""Get a circuit breaker by name"""
return self._breakers.get(name)
def protect(self, name: str) -> Callable:
"""Decorator to protect a function with a circuit breaker"""
def decorator(func: Callable) -> Callable:
breaker = self.register(name)
return breaker(func)
return decorator
def get_all_status(self) -> dict:
"""Get status of all circuit breakers"""
return {
name: breaker.to_dict()
for name, breaker in self._breakers.items()
}
def reset_all(self):
"""Reset all circuit breakers"""
for breaker in self._breakers.values():
breaker.reset()
def persist(self):
"""Persist circuit breaker states to Redis"""
if not self._redis:
return
try:
data = json.dumps(self.get_all_status())
self._redis.set("circuit_breakers:status", data)
except Exception:
pass
# =============================================================================
# Pre-configured Circuit Breakers for Governance System
# =============================================================================
# Global registry
_registry = CircuitBreakerRegistry()
def get_vault_breaker() -> CircuitBreaker:
"""Get circuit breaker for Vault operations"""
return _registry.register("vault", CircuitConfig(
name="vault",
failure_threshold=3,
success_threshold=2,
timeout_seconds=30.0,
call_timeout=10.0
))
def get_dragonfly_breaker() -> CircuitBreaker:
"""Get circuit breaker for DragonflyDB operations"""
return _registry.register("dragonfly", CircuitConfig(
name="dragonfly",
failure_threshold=5,
success_threshold=3,
timeout_seconds=15.0,
call_timeout=5.0
))
def get_ledger_breaker() -> CircuitBreaker:
"""Get circuit breaker for Ledger operations"""
return _registry.register("ledger", CircuitConfig(
name="ledger",
failure_threshold=3,
success_threshold=2,
timeout_seconds=10.0,
call_timeout=5.0
))
def get_registry() -> CircuitBreakerRegistry:
"""Get the global circuit breaker registry"""
return _registry
# =============================================================================
# CLI Interface
# =============================================================================
def main():
import argparse
parser = argparse.ArgumentParser(description="Circuit Breaker CLI")
parser.add_argument("command", choices=["status", "reset", "test"],
help="Command to run")
parser.add_argument("--name", type=str, help="Circuit breaker name")
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
# Initialize default breakers
get_vault_breaker()
get_dragonfly_breaker()
get_ledger_breaker()
if args.command == "status":
status = _registry.get_all_status()
if args.json:
print(json.dumps(status, indent=2))
else:
print("\nCircuit Breaker Status")
print("=" * 50)
for name, data in status.items():
state_icon = {
"closed": "✅",
"open": "❌",
"half_open": "⚠️"
}.get(data["state"], "?")
stats = data["stats"]
print(f"\n{state_icon} {name}: {data['state'].upper()}")
print(f" Calls: {stats['total_calls']} total, {stats['successful_calls']} success, {stats['failed_calls']} failed")
print(f" Success Rate: {stats['success_rate']*100:.1f}%")
print(f" Consecutive: {stats['consecutive_failures']} failures, {stats['consecutive_successes']} successes")
elif args.command == "reset":
if args.name:
breaker = _registry.get(args.name)
if breaker:
breaker.reset()
print(f"Reset circuit breaker: {args.name}")
else:
print(f"Circuit breaker not found: {args.name}")
else:
_registry.reset_all()
print("Reset all circuit breakers")
elif args.command == "test":
print("\nTesting circuit breaker behavior...")
# Create a test breaker
test_breaker = CircuitBreaker(CircuitConfig(
name="test",
failure_threshold=3,
success_threshold=2,
timeout_seconds=5.0
))
# Simulate failures
print("\n1. Simulating 3 failures to trip circuit:")
for i in range(3):
try:
@test_breaker
def failing_call():
raise Exception("Simulated failure")
failing_call()
except Exception as e:
print(f" Call {i+1}: Failed - {e}")
print(f"\n Circuit state: {test_breaker.state.value}")
# Try a call when open
print("\n2. Trying call when circuit is open:")
try:
@test_breaker
def blocked_call():
return "success"
blocked_call()
except CircuitOpenError as e:
print(f" Rejected: {e}")
print(f"\n Stats: {test_breaker.stats.to_dict()}")
print("\nTest complete!")
if __name__ == "__main__":
main()
+712
View File
@@ -0,0 +1,712 @@
#!/usr/bin/env python3
"""
Agent Runtime Governance via DragonflyDB
=========================================
Implements real-time agent control: instruction packets, state tracking,
error budgets, revocation, and handoffs.
"""
import json
import hashlib
import time
from dataclasses import dataclass, field, asdict
from datetime import datetime, timezone
from enum import Enum
from typing import Any, Optional
import redis
import subprocess
# =============================================================================
# Configuration
# =============================================================================
def get_dragonfly_client() -> redis.Redis:
"""Get DragonflyDB client with credentials from Vault"""
# Get credentials from Vault
with open("/opt/vault/init-keys.json") as f:
token = json.load(f)["root_token"]
result = subprocess.run([
"curl", "-sk",
"-H", f"X-Vault-Token: {token}",
"https://127.0.0.1:8200/v1/secret/data/services/dragonfly"
], capture_output=True, text=True)
creds = json.loads(result.stdout)["data"]["data"]
return redis.Redis(
host=creds["host"],
port=int(creds["port"]),
password=creds["password"],
decode_responses=True
)
# =============================================================================
# Import Core Definitions (from unified pipeline.core module)
# =============================================================================
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from pipeline.core import (
AgentPhase,
AgentStatus,
RevocationType,
ErrorBudget,
AGENT_PHASE_NAMES,
AGENT_PHASES_ORDERED,
RedisKeys,
DEFAULT_HEARTBEAT_TTL,
DEFAULT_LOCK_TTL,
)
@dataclass
class InstructionPacket:
agent_id: str
task_id: str
created_for: str
objective: str
deliverables: list[str]
constraints: dict
success_criteria: list[str]
error_budget: ErrorBudget
escalation_rules: list[str]
repo: str = ""
pr_context: dict = field(default_factory=dict)
revocation_context: dict = field(default_factory=dict)
created_at: str = ""
def __post_init__(self):
if not self.created_at:
self.created_at = datetime.now(timezone.utc).isoformat()
if isinstance(self.error_budget, dict):
self.error_budget = ErrorBudget(**self.error_budget)
def to_dict(self) -> dict:
d = asdict(self)
d["error_budget"] = asdict(self.error_budget)
return d
@dataclass
class AgentState:
agent_id: str
status: AgentStatus
phase: AgentPhase
step: str = ""
started_at: str = ""
last_progress_at: str = ""
current_pr: Optional[int] = None
notes: str = ""
def to_dict(self) -> dict:
return {
"agent_id": self.agent_id,
"status": self.status.value,
"phase": self.phase.value,
"step": self.step,
"started_at": self.started_at,
"last_progress_at": self.last_progress_at,
"current_pr": self.current_pr,
"notes": self.notes
}
@dataclass
class HandoffObject:
task_id: str
previous_agent_id: str
revoked: bool
revocation_reason: dict
last_known_state: dict
what_was_tried: list[str]
blocking_issue: str
required_next_actions: list[str]
constraints_reminder: list[str]
artifacts: list[str]
created_at: str = ""
def __post_init__(self):
if not self.created_at:
self.created_at = datetime.now(timezone.utc).isoformat()
def to_dict(self) -> dict:
return asdict(self)
# =============================================================================
# Governance Manager
# =============================================================================
class GovernanceManager:
"""
Core runtime governance via DragonflyDB.
Manages instruction packets, state, errors, locks, and handoffs.
"""
def __init__(self):
self.db = get_dragonfly_client()
self.lock_ttl = 300 # 5 minutes default lock TTL
self.heartbeat_ttl = 60 # 1 minute heartbeat timeout
def _now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _error_signature(self, error_type: str, message: str) -> str:
"""Generate a normalized error signature"""
normalized = f"{error_type}:{message[:100]}".lower()
return hashlib.md5(normalized.encode()).hexdigest()[:12]
# -------------------------------------------------------------------------
# Instruction Packets
# -------------------------------------------------------------------------
def create_instruction_packet(self, packet: InstructionPacket) -> bool:
"""Store an instruction packet for an agent"""
key = f"agent:{packet.agent_id}:packet"
return self.db.set(key, json.dumps(packet.to_dict()))
def get_instruction_packet(self, agent_id: str) -> Optional[InstructionPacket]:
"""Retrieve an agent's instruction packet"""
key = f"agent:{agent_id}:packet"
data = self.db.get(key)
if data:
return InstructionPacket(**json.loads(data))
return None
# -------------------------------------------------------------------------
# Agent State
# -------------------------------------------------------------------------
def set_agent_state(self, state: AgentState) -> bool:
"""Update agent's runtime state"""
state.last_progress_at = self._now()
key = f"agent:{state.agent_id}:state"
return self.db.set(key, json.dumps(state.to_dict()))
def get_agent_state(self, agent_id: str) -> Optional[AgentState]:
"""Get agent's current state"""
key = f"agent:{agent_id}:state"
data = self.db.get(key)
if data:
d = json.loads(data)
return AgentState(
agent_id=d["agent_id"],
status=AgentStatus(d["status"]),
phase=AgentPhase(d["phase"]),
step=d.get("step", ""),
started_at=d.get("started_at", ""),
last_progress_at=d.get("last_progress_at", ""),
current_pr=d.get("current_pr"),
notes=d.get("notes", "")
)
return None
def transition_phase(self, agent_id: str, phase: AgentPhase, step: str = "", notes: str = "") -> bool:
"""Transition agent to a new phase"""
state = self.get_agent_state(agent_id)
if not state:
return False
state.phase = phase
state.step = step
state.notes = notes
return self.set_agent_state(state)
# -------------------------------------------------------------------------
# Locking
# -------------------------------------------------------------------------
def acquire_lock(self, agent_id: str, ttl: int = None) -> bool:
"""Acquire execution lock for an agent"""
key = f"agent:{agent_id}:lock"
ttl = ttl or self.lock_ttl
# NX = only set if not exists
return self.db.set(key, self._now(), nx=True, ex=ttl)
def refresh_lock(self, agent_id: str, ttl: int = None) -> bool:
"""Refresh an existing lock"""
key = f"agent:{agent_id}:lock"
ttl = ttl or self.lock_ttl
if self.db.exists(key):
return self.db.expire(key, ttl)
return False
def release_lock(self, agent_id: str) -> bool:
"""Release execution lock"""
key = f"agent:{agent_id}:lock"
return self.db.delete(key) > 0
def has_lock(self, agent_id: str) -> bool:
"""Check if agent has a valid lock"""
key = f"agent:{agent_id}:lock"
return self.db.exists(key)
# -------------------------------------------------------------------------
# Heartbeat
# -------------------------------------------------------------------------
def heartbeat(self, agent_id: str) -> bool:
"""Update agent heartbeat"""
key = f"agent:{agent_id}:heartbeat"
return self.db.set(key, self._now(), ex=self.heartbeat_ttl)
def is_alive(self, agent_id: str) -> bool:
"""Check if agent heartbeat is fresh"""
key = f"agent:{agent_id}:heartbeat"
return self.db.exists(key)
# -------------------------------------------------------------------------
# Error Tracking
# -------------------------------------------------------------------------
def record_error(self, agent_id: str, error_type: str, message: str) -> dict:
"""Record an error and return current counts"""
key = f"agent:{agent_id}:errors"
sig = self._error_signature(error_type, message)
pipe = self.db.pipeline()
pipe.hincrby(key, "total_errors", 1)
pipe.hincrby(key, f"same_error:{sig}", 1)
pipe.hset(key, "last_error_signature", sig)
pipe.hset(key, "last_error_at", self._now())
pipe.hset(key, "last_error_type", error_type)
pipe.hset(key, "last_error_message", message[:500])
pipe.execute()
return self.get_error_counts(agent_id)
def record_procedure_violation(self, agent_id: str, violation: str) -> int:
"""Record a procedure violation"""
key = f"agent:{agent_id}:errors"
self.db.hset(key, "last_violation", violation)
self.db.hset(key, "last_violation_at", self._now())
return self.db.hincrby(key, "procedure_violations", 1)
def get_error_counts(self, agent_id: str) -> dict:
"""Get all error counts for an agent"""
key = f"agent:{agent_id}:errors"
data = self.db.hgetall(key)
return {
"total_errors": int(data.get("total_errors", 0)),
"procedure_violations": int(data.get("procedure_violations", 0)),
"last_error_signature": data.get("last_error_signature", ""),
"last_error_at": data.get("last_error_at", ""),
"last_error_type": data.get("last_error_type", ""),
"same_error_counts": {
k.replace("same_error:", ""): int(v)
for k, v in data.items()
if k.startswith("same_error:")
}
}
def check_error_budget(self, agent_id: str) -> tuple[bool, Optional[str]]:
"""Check if error budget is exceeded. Returns (ok, reason)"""
packet = self.get_instruction_packet(agent_id)
if not packet:
return False, "NO_INSTRUCTION_PACKET"
counts = self.get_error_counts(agent_id)
budget = packet.error_budget
# Check procedure violations
if counts["procedure_violations"] >= budget.max_procedure_violations:
return False, f"PROCEDURE_VIOLATIONS ({counts['procedure_violations']} >= {budget.max_procedure_violations})"
# Check total errors
if counts["total_errors"] >= budget.max_total_errors:
return False, f"TOTAL_ERRORS ({counts['total_errors']} >= {budget.max_total_errors})"
# Check same error repeats
for sig, count in counts["same_error_counts"].items():
if count >= budget.max_same_error_repeats:
return False, f"SAME_ERROR_REPEATED ({sig}: {count} >= {budget.max_same_error_repeats})"
return True, None
# -------------------------------------------------------------------------
# Task Management
# -------------------------------------------------------------------------
def assign_agent_to_task(self, task_id: str, agent_id: str) -> bool:
"""Assign an agent to a task"""
# Set active agent
self.db.set(f"task:{task_id}:active_agent", agent_id)
# Add to history
self.db.rpush(f"task:{task_id}:history", json.dumps({
"agent_id": agent_id,
"assigned_at": self._now(),
"event": "ASSIGNED"
}))
return True
def get_active_agent(self, task_id: str) -> Optional[str]:
"""Get the currently active agent for a task"""
return self.db.get(f"task:{task_id}:active_agent")
def get_task_history(self, task_id: str) -> list[dict]:
"""Get task agent history"""
data = self.db.lrange(f"task:{task_id}:history", 0, -1)
return [json.loads(d) for d in data]
# -------------------------------------------------------------------------
# Revocation
# -------------------------------------------------------------------------
def revoke_agent(self, agent_id: str, reason_type: RevocationType, details: str) -> bool:
"""Revoke an agent's access"""
# Update state
state = self.get_agent_state(agent_id)
if state:
state.status = AgentStatus.REVOKED
state.phase = AgentPhase.REVOKED
state.notes = f"Revoked: {reason_type.value} - {details}"
self.set_agent_state(state)
# Release lock
self.release_lock(agent_id)
# Write to revocation ledger
revocation_event = {
"agent_id": agent_id,
"reason_type": reason_type.value,
"details": details,
"revoked_at": self._now()
}
self.db.rpush("revocations:ledger", json.dumps(revocation_event))
# Add to task history if we know the task
packet = self.get_instruction_packet(agent_id)
if packet:
self.db.rpush(f"task:{packet.task_id}:history", json.dumps({
"agent_id": agent_id,
"event": "REVOKED",
"reason": reason_type.value,
"revoked_at": self._now()
}))
return True
def get_recent_revocations(self, count: int = 50) -> list[dict]:
"""Get recent revocation events"""
data = self.db.lrange("revocations:ledger", -count, -1)
return [json.loads(d) for d in data]
# -------------------------------------------------------------------------
# Handoff
# -------------------------------------------------------------------------
def create_handoff(self, handoff: HandoffObject) -> bool:
"""Create a handoff object for task continuity"""
key = f"handoff:{handoff.task_id}:latest"
return self.db.set(key, json.dumps(handoff.to_dict()))
def get_handoff(self, task_id: str) -> Optional[HandoffObject]:
"""Get the latest handoff for a task"""
key = f"handoff:{task_id}:latest"
data = self.db.get(key)
if data:
return HandoffObject(**json.loads(data))
return None
# -------------------------------------------------------------------------
# Artifacts
# -------------------------------------------------------------------------
def register_artifact(self, task_id: str, artifact_type: str, reference: str) -> bool:
"""Register an artifact for a task"""
key = f"task:{task_id}:artifacts"
artifact = {
"type": artifact_type,
"reference": reference,
"created_at": self._now()
}
self.db.rpush(key, json.dumps(artifact))
return True
def get_artifacts(self, task_id: str) -> list[dict]:
"""Get all artifacts for a task"""
key = f"task:{task_id}:artifacts"
data = self.db.lrange(key, 0, -1)
return [json.loads(d) for d in data]
def has_required_artifact(self, task_id: str, artifact_type: str) -> bool:
"""Check if a required artifact exists"""
artifacts = self.get_artifacts(task_id)
return any(a["type"] == artifact_type for a in artifacts)
# =============================================================================
# Worker Agent Runtime
# =============================================================================
class WorkerRuntime:
"""
Runtime for worker agents.
Handles bootstrap, state transitions, error reporting, and compliance.
"""
def __init__(self, agent_id: str):
self.agent_id = agent_id
self.gov = GovernanceManager()
self.packet: Optional[InstructionPacket] = None
self.state: Optional[AgentState] = None
def bootstrap(self) -> tuple[bool, str]:
"""
Bootstrap the agent runtime.
Returns (success, message)
"""
# Step 1: Read revocation ledger
revocations = self.gov.get_recent_revocations(50)
if revocations:
print(f"[BOOTSTRAP] Read {len(revocations)} recent revocations")
# Check if this agent was previously revoked
for rev in revocations:
if rev["agent_id"] == self.agent_id:
return False, f"AGENT_PREVIOUSLY_REVOKED: {rev['reason_type']}"
# Step 2: Read instruction packet
self.packet = self.gov.get_instruction_packet(self.agent_id)
if not self.packet:
return False, "NO_INSTRUCTION_PACKET"
print(f"[BOOTSTRAP] Loaded instruction packet for task: {self.packet.task_id}")
# Step 3: Check for existing handoff
handoff = self.gov.get_handoff(self.packet.task_id)
if handoff and handoff.previous_agent_id != self.agent_id:
print(f"[BOOTSTRAP] Found handoff from previous agent: {handoff.previous_agent_id}")
print(f"[BOOTSTRAP] Revocation reason: {handoff.revocation_reason}")
print(f"[BOOTSTRAP] Required next actions: {handoff.required_next_actions}")
# Step 4: Acquire lock
if not self.gov.acquire_lock(self.agent_id):
return False, "CANNOT_ACQUIRE_LOCK"
# Step 5: Initialize state
self.state = AgentState(
agent_id=self.agent_id,
status=AgentStatus.RUNNING,
phase=AgentPhase.BOOTSTRAP,
step="initialized",
started_at=datetime.now(timezone.utc).isoformat()
)
self.gov.set_agent_state(self.state)
# Step 6: Start heartbeat
self.gov.heartbeat(self.agent_id)
# Step 7: Assign to task
self.gov.assign_agent_to_task(self.packet.task_id, self.agent_id)
return True, "BOOTSTRAP_COMPLETE"
def transition(self, phase: AgentPhase, step: str = "", notes: str = "") -> bool:
"""Transition to a new phase"""
if not self.state:
return False
# Refresh heartbeat and lock
self.gov.heartbeat(self.agent_id)
self.gov.refresh_lock(self.agent_id)
# Check error budget before transition
ok, reason = self.gov.check_error_budget(self.agent_id)
if not ok:
self.gov.revoke_agent(self.agent_id, RevocationType.ERROR_BUDGET_EXCEEDED, reason)
return False
# Update state
self.state.phase = phase
self.state.step = step
self.state.notes = notes
self.gov.set_agent_state(self.state)
print(f"[PHASE] {phase.value} - {step}")
return True
def report_error(self, error_type: str, message: str) -> bool:
"""Report an error and check if we should continue"""
counts = self.gov.record_error(self.agent_id, error_type, message)
print(f"[ERROR] {error_type}: {message}")
print(f"[ERROR] Counts: total={counts['total_errors']}, violations={counts['procedure_violations']}")
ok, reason = self.gov.check_error_budget(self.agent_id)
if not ok:
print(f"[REVOKE] Error budget exceeded: {reason}")
self.gov.revoke_agent(self.agent_id, RevocationType.ERROR_BUDGET_EXCEEDED, reason)
return False
return True
def report_violation(self, violation: str) -> bool:
"""Report a procedure violation"""
count = self.gov.record_procedure_violation(self.agent_id, violation)
print(f"[VIOLATION] {violation} (count: {count})")
ok, reason = self.gov.check_error_budget(self.agent_id)
if not ok:
print(f"[REVOKE] Procedure violation limit: {reason}")
self.gov.revoke_agent(self.agent_id, RevocationType.PROCEDURE_VIOLATION, reason)
return False
return True
def register_artifact(self, artifact_type: str, reference: str):
"""Register a work artifact"""
if self.packet:
self.gov.register_artifact(self.packet.task_id, artifact_type, reference)
print(f"[ARTIFACT] Registered: {artifact_type} -> {reference}")
def complete(self, notes: str = "") -> bool:
"""Mark work as complete"""
if self.state:
self.state.status = AgentStatus.COMPLETED
self.state.phase = AgentPhase.EXIT
self.state.notes = notes
self.gov.set_agent_state(self.state)
self.gov.release_lock(self.agent_id)
print(f"[COMPLETE] {notes}")
return True
def create_handoff(self, blocking_issue: str, what_was_tried: list[str],
required_next_actions: list[str]) -> bool:
"""Create a handoff for the next agent"""
if not self.packet or not self.state:
return False
handoff = HandoffObject(
task_id=self.packet.task_id,
previous_agent_id=self.agent_id,
revoked=self.state.status == AgentStatus.REVOKED,
revocation_reason={
"type": self.state.status.value,
"details": self.state.notes
},
last_known_state={
"phase": self.state.phase.value,
"step": self.state.step
},
what_was_tried=what_was_tried,
blocking_issue=blocking_issue,
required_next_actions=required_next_actions,
constraints_reminder=self.packet.constraints.get("forbidden", []),
artifacts=[a["reference"] for a in self.gov.get_artifacts(self.packet.task_id)]
)
return self.gov.create_handoff(handoff)
# =============================================================================
# CLI
# =============================================================================
if __name__ == "__main__":
import sys
gov = GovernanceManager()
if len(sys.argv) < 2:
print("Usage: governance.py <command> [args]")
print("Commands:")
print(" create-packet <agent_id> <task_id> <objective>")
print(" get-state <agent_id>")
print(" get-errors <agent_id>")
print(" revocations")
print(" test")
sys.exit(1)
cmd = sys.argv[1]
if cmd == "create-packet":
agent_id, task_id, objective = sys.argv[2], sys.argv[3], sys.argv[4]
packet = InstructionPacket(
agent_id=agent_id,
task_id=task_id,
created_for="CLI test",
objective=objective,
deliverables=["plan artifacts", "run logs"],
constraints={
"scope": ["sandbox only"],
"forbidden": ["no prod access", "no unrecorded root"],
"required_steps": ["plan before apply"]
},
success_criteria=["CI passes", "artifacts uploaded"],
error_budget=ErrorBudget(),
escalation_rules=["If blocked > 20m -> escalate"]
)
gov.create_instruction_packet(packet)
print(f"Created packet for {agent_id}")
elif cmd == "get-state":
agent_id = sys.argv[2]
state = gov.get_agent_state(agent_id)
if state:
print(json.dumps(state.to_dict(), indent=2))
else:
print("No state found")
elif cmd == "get-errors":
agent_id = sys.argv[2]
errors = gov.get_error_counts(agent_id)
print(json.dumps(errors, indent=2))
elif cmd == "revocations":
revs = gov.get_recent_revocations()
for r in revs:
print(json.dumps(r))
elif cmd == "test":
print("=== Testing Governance System ===\n")
# Create test packet
packet = InstructionPacket(
agent_id="test-agent-001",
task_id="test-task-001",
created_for="Governance test",
objective="Test the governance system",
deliverables=["test output"],
constraints={
"scope": ["sandbox"],
"forbidden": ["prod access"],
"required_steps": ["plan first"]
},
success_criteria=["test passes"],
error_budget=ErrorBudget(max_total_errors=5, max_same_error_repeats=2),
escalation_rules=[]
)
gov.create_instruction_packet(packet)
print("[OK] Created instruction packet")
# Test worker runtime
worker = WorkerRuntime("test-agent-001")
ok, msg = worker.bootstrap()
print(f"[OK] Bootstrap: {msg}")
# Simulate work phases
worker.transition(AgentPhase.PREFLIGHT, "scope_check")
worker.transition(AgentPhase.PLAN, "generating_plan")
worker.register_artifact("plan", "plan_output_001")
worker.transition(AgentPhase.EXECUTE, "applying")
# Simulate some errors
worker.report_error("VALIDATION_ERROR", "Variable not defined")
worker.report_error("VALIDATION_ERROR", "Variable not defined") # Same error
# Check error budget
ok, reason = gov.check_error_budget("test-agent-001")
print(f"[OK] Error budget check: ok={ok}, reason={reason}")
# Complete
worker.complete("Test completed successfully")
print("\n=== Test Complete ===")
else:
print(f"Unknown command: {cmd}")
+448
View File
@@ -0,0 +1,448 @@
#!/usr/bin/env python3
"""
Health Manager
==============
Production health check infrastructure for monitoring system dependencies.
Monitors:
- HashiCorp Vault (authentication, policies)
- DragonflyDB (state management)
- SQLite Ledger (audit trail)
- Agent processes
Part of Phase 8: Production Hardening.
"""
import json
import sqlite3
import subprocess
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from pathlib import Path
from typing import Optional, Callable
import redis
class HealthStatus(str, Enum):
"""Health check status"""
HEALTHY = "healthy"
DEGRADED = "degraded"
UNHEALTHY = "unhealthy"
UNKNOWN = "unknown"
class DependencyType(str, Enum):
"""Types of system dependencies"""
VAULT = "vault"
DRAGONFLY = "dragonfly"
LEDGER = "ledger"
AGENT = "agent"
NETWORK = "network"
@dataclass
class HealthCheckResult:
"""Result of a health check"""
dependency: DependencyType
status: HealthStatus
latency_ms: float
message: str
checked_at: str = ""
details: dict = field(default_factory=dict)
def __post_init__(self):
if not self.checked_at:
self.checked_at = datetime.now(timezone.utc).isoformat()
def to_dict(self) -> dict:
return {
"dependency": self.dependency.value,
"status": self.status.value,
"latency_ms": self.latency_ms,
"message": self.message,
"checked_at": self.checked_at,
"details": self.details
}
@dataclass
class SystemHealth:
"""Overall system health status"""
status: HealthStatus
checks: list[HealthCheckResult]
healthy_count: int
degraded_count: int
unhealthy_count: int
checked_at: str = ""
def __post_init__(self):
if not self.checked_at:
self.checked_at = datetime.now(timezone.utc).isoformat()
def to_dict(self) -> dict:
return {
"status": self.status.value,
"healthy": self.healthy_count,
"degraded": self.degraded_count,
"unhealthy": self.unhealthy_count,
"checked_at": self.checked_at,
"checks": [c.to_dict() for c in self.checks]
}
class HealthManager:
"""
Manages health checks for all system dependencies.
Features:
- Individual dependency health checks
- Aggregate system health
- Configurable thresholds
- Health history tracking
"""
# Default configuration
VAULT_ADDR = "https://127.0.0.1:8200"
VAULT_TOKEN_FILE = "/opt/vault/init-keys.json"
REDIS_HOST = "127.0.0.1"
REDIS_PORT = 6379
LEDGER_PATH = "/opt/agent-governance/ledger/governance.db"
# Thresholds (ms)
LATENCY_WARN_THRESHOLD = 500
LATENCY_CRITICAL_THRESHOLD = 2000
CHECK_TIMEOUT = 5
def __init__(self, base_path: str = "/opt/agent-governance"):
self.base_path = Path(base_path)
self._vault_token: Optional[str] = None
self._redis: Optional[redis.Redis] = None
self._history: list[SystemHealth] = []
def _now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _get_vault_token(self) -> str:
"""Get Vault root token"""
if self._vault_token:
return self._vault_token
try:
with open(self.VAULT_TOKEN_FILE) as f:
self._vault_token = json.load(f)["root_token"]
return self._vault_token
except Exception:
return ""
def _get_redis(self) -> Optional[redis.Redis]:
"""Get Redis/DragonflyDB client"""
if self._redis:
return self._redis
try:
# Get credentials from Vault
token = self._get_vault_token()
result = subprocess.run([
"curl", "-sk",
"-H", f"X-Vault-Token: {token}",
f"{self.VAULT_ADDR}/v1/secret/data/services/dragonfly"
], capture_output=True, text=True, timeout=self.CHECK_TIMEOUT)
creds = json.loads(result.stdout)["data"]["data"]
self._redis = redis.Redis(
host=creds["host"],
port=int(creds["port"]),
password=creds["password"],
decode_responses=True,
socket_timeout=self.CHECK_TIMEOUT
)
return self._redis
except Exception:
return None
def check_vault(self) -> HealthCheckResult:
"""Check Vault health"""
start = time.time()
try:
token = self._get_vault_token()
if not token:
return HealthCheckResult(
dependency=DependencyType.VAULT,
status=HealthStatus.UNHEALTHY,
latency_ms=0,
message="Cannot read Vault token"
)
result = subprocess.run([
"curl", "-sk",
"-H", f"X-Vault-Token: {token}",
f"{self.VAULT_ADDR}/v1/sys/health"
], capture_output=True, text=True, timeout=self.CHECK_TIMEOUT)
latency = (time.time() - start) * 1000
data = json.loads(result.stdout)
if data.get("sealed"):
return HealthCheckResult(
dependency=DependencyType.VAULT,
status=HealthStatus.UNHEALTHY,
latency_ms=latency,
message="Vault is sealed",
details=data
)
if not data.get("initialized"):
return HealthCheckResult(
dependency=DependencyType.VAULT,
status=HealthStatus.UNHEALTHY,
latency_ms=latency,
message="Vault is not initialized",
details=data
)
status = HealthStatus.HEALTHY
if latency > self.LATENCY_CRITICAL_THRESHOLD:
status = HealthStatus.DEGRADED
elif latency > self.LATENCY_WARN_THRESHOLD:
status = HealthStatus.DEGRADED
return HealthCheckResult(
dependency=DependencyType.VAULT,
status=status,
latency_ms=latency,
message="Vault is healthy",
details={"initialized": True, "sealed": False}
)
except subprocess.TimeoutExpired:
return HealthCheckResult(
dependency=DependencyType.VAULT,
status=HealthStatus.UNHEALTHY,
latency_ms=self.CHECK_TIMEOUT * 1000,
message="Vault health check timed out"
)
except Exception as e:
return HealthCheckResult(
dependency=DependencyType.VAULT,
status=HealthStatus.UNHEALTHY,
latency_ms=(time.time() - start) * 1000,
message=f"Vault check failed: {str(e)}"
)
def check_dragonfly(self) -> HealthCheckResult:
"""Check DragonflyDB health"""
start = time.time()
try:
r = self._get_redis()
if not r:
return HealthCheckResult(
dependency=DependencyType.DRAGONFLY,
status=HealthStatus.UNHEALTHY,
latency_ms=0,
message="Cannot connect to DragonflyDB"
)
# Ping test
r.ping()
latency = (time.time() - start) * 1000
# Get info
info = r.info("server")
status = HealthStatus.HEALTHY
if latency > self.LATENCY_CRITICAL_THRESHOLD:
status = HealthStatus.DEGRADED
elif latency > self.LATENCY_WARN_THRESHOLD:
status = HealthStatus.DEGRADED
return HealthCheckResult(
dependency=DependencyType.DRAGONFLY,
status=status,
latency_ms=latency,
message="DragonflyDB is healthy",
details={
"version": info.get("redis_version", "unknown"),
"uptime_seconds": info.get("uptime_in_seconds", 0)
}
)
except redis.ConnectionError:
return HealthCheckResult(
dependency=DependencyType.DRAGONFLY,
status=HealthStatus.UNHEALTHY,
latency_ms=(time.time() - start) * 1000,
message="DragonflyDB connection refused"
)
except Exception as e:
return HealthCheckResult(
dependency=DependencyType.DRAGONFLY,
status=HealthStatus.UNHEALTHY,
latency_ms=(time.time() - start) * 1000,
message=f"DragonflyDB check failed: {str(e)}"
)
def check_ledger(self) -> HealthCheckResult:
"""Check SQLite Ledger health"""
start = time.time()
try:
if not Path(self.LEDGER_PATH).exists():
return HealthCheckResult(
dependency=DependencyType.LEDGER,
status=HealthStatus.UNHEALTHY,
latency_ms=0,
message="Ledger database file not found"
)
conn = sqlite3.connect(self.LEDGER_PATH, timeout=self.CHECK_TIMEOUT)
cursor = conn.cursor()
# Check tables exist
cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
tables = [row[0] for row in cursor.fetchall()]
required_tables = ["agent_metrics", "violations", "promotions", "orchestration_log"]
missing = [t for t in required_tables if t not in tables]
latency = (time.time() - start) * 1000
conn.close()
if missing:
return HealthCheckResult(
dependency=DependencyType.LEDGER,
status=HealthStatus.DEGRADED,
latency_ms=latency,
message=f"Missing tables: {', '.join(missing)}",
details={"tables": tables, "missing": missing}
)
status = HealthStatus.HEALTHY
if latency > self.LATENCY_WARN_THRESHOLD:
status = HealthStatus.DEGRADED
return HealthCheckResult(
dependency=DependencyType.LEDGER,
status=status,
latency_ms=latency,
message="Ledger is healthy",
details={"tables": tables}
)
except sqlite3.OperationalError as e:
return HealthCheckResult(
dependency=DependencyType.LEDGER,
status=HealthStatus.UNHEALTHY,
latency_ms=(time.time() - start) * 1000,
message=f"Ledger error: {str(e)}"
)
except Exception as e:
return HealthCheckResult(
dependency=DependencyType.LEDGER,
status=HealthStatus.UNHEALTHY,
latency_ms=(time.time() - start) * 1000,
message=f"Ledger check failed: {str(e)}"
)
def check_all(self) -> SystemHealth:
"""Run all health checks and return aggregate status"""
checks = [
self.check_vault(),
self.check_dragonfly(),
self.check_ledger()
]
healthy = sum(1 for c in checks if c.status == HealthStatus.HEALTHY)
degraded = sum(1 for c in checks if c.status == HealthStatus.DEGRADED)
unhealthy = sum(1 for c in checks if c.status == HealthStatus.UNHEALTHY)
# Determine overall status
if unhealthy > 0:
overall = HealthStatus.UNHEALTHY
elif degraded > 0:
overall = HealthStatus.DEGRADED
else:
overall = HealthStatus.HEALTHY
health = SystemHealth(
status=overall,
checks=checks,
healthy_count=healthy,
degraded_count=degraded,
unhealthy_count=unhealthy
)
# Track history
self._history.append(health)
if len(self._history) > 100:
self._history = self._history[-100:]
# Store in DragonflyDB
try:
r = self._get_redis()
if r:
r.set("health:latest", json.dumps(health.to_dict()))
r.lpush("health:history", json.dumps(health.to_dict()))
r.ltrim("health:history", 0, 99)
except Exception:
pass
return health
def get_status(self) -> dict:
"""Get current health status as dict"""
health = self.check_all()
return health.to_dict()
def is_healthy(self) -> bool:
"""Quick check if system is healthy"""
health = self.check_all()
return health.status == HealthStatus.HEALTHY
# =============================================================================
# CLI Interface
# =============================================================================
def main():
import argparse
parser = argparse.ArgumentParser(description="Health Manager CLI")
parser.add_argument("command", choices=["check", "status", "watch"],
help="Command to run")
parser.add_argument("--json", action="store_true", help="Output as JSON")
parser.add_argument("--interval", type=int, default=5,
help="Watch interval in seconds")
args = parser.parse_args()
manager = HealthManager()
if args.command == "check":
health = manager.check_all()
if args.json:
print(json.dumps(health.to_dict(), indent=2))
else:
print(f"\nSystem Health: {health.status.value.upper()}")
print(f"Healthy: {health.healthy_count} | Degraded: {health.degraded_count} | Unhealthy: {health.unhealthy_count}")
print()
for check in health.checks:
icon = {"healthy": "✅", "degraded": "⚠️", "unhealthy": "❌"}.get(check.status.value, "?")
print(f" {icon} {check.dependency.value}: {check.message} ({check.latency_ms:.1f}ms)")
elif args.command == "status":
health = manager.check_all()
print("HEALTHY" if health.status == HealthStatus.HEALTHY else health.status.value.upper())
elif args.command == "watch":
print("Watching health (Ctrl+C to stop)...")
try:
while True:
health = manager.check_all()
timestamp = datetime.now().strftime("%H:%M:%S")
status_icon = {"healthy": "✅", "degraded": "⚠️", "unhealthy": "❌"}.get(health.status.value, "?")
print(f"[{timestamp}] {status_icon} {health.status.value} - V:{health.checks[0].latency_ms:.0f}ms D:{health.checks[1].latency_ms:.0f}ms L:{health.checks[2].latency_ms:.0f}ms")
time.sleep(args.interval)
except KeyboardInterrupt:
print("\nStopped.")
if __name__ == "__main__":
main()
+317
View File
@@ -0,0 +1,317 @@
#!/usr/bin/env python3
"""
Governance Monitor Agents
=========================
These agents watch and enforce, they don't do work.
- Execution Watcher: Heartbeats, stuck detection, lock expiry
- Compliance Watcher: Artifact checks, forbidden action detection
- Vault Lease Watcher: Token validity, revocation enforcement
"""
import json
import time
from datetime import datetime, timezone, timedelta
from typing import Optional
import threading
from governance import (
GovernanceManager,
AgentPhase,
AgentStatus,
RevocationType
)
class ExecutionWatcher:
"""
Monitors agent execution health.
- Checks heartbeats
- Detects stuck agents
- Handles lock expiry
"""
def __init__(self, heartbeat_timeout: int = 60, stuck_threshold: int = 300):
self.gov = GovernanceManager()
self.heartbeat_timeout = heartbeat_timeout # seconds
self.stuck_threshold = stuck_threshold # seconds without progress
def check_agent(self, agent_id: str) -> dict:
"""Check an agent's execution health"""
result = {
"agent_id": agent_id,
"healthy": True,
"issues": []
}
state = self.gov.get_agent_state(agent_id)
if not state:
result["healthy"] = False
result["issues"].append("NO_STATE")
return result
# Skip completed/revoked agents
if state.status in [AgentStatus.COMPLETED, AgentStatus.REVOKED]:
return result
# Check heartbeat
if not self.gov.is_alive(agent_id):
result["healthy"] = False
result["issues"].append("HEARTBEAT_TIMEOUT")
# Check lock
if not self.gov.has_lock(agent_id) and state.status == AgentStatus.RUNNING:
result["healthy"] = False
result["issues"].append("LOCK_EXPIRED")
# Check for stuck (no progress)
if state.last_progress_at:
last_progress = datetime.fromisoformat(state.last_progress_at.replace("Z", "+00:00"))
age = (datetime.now(timezone.utc) - last_progress).total_seconds()
if age > self.stuck_threshold:
result["healthy"] = False
result["issues"].append(f"STUCK_{int(age)}s")
return result
def enforce(self, agent_id: str) -> Optional[str]:
"""Enforce health requirements, return action taken"""
check = self.check_agent(agent_id)
if check["healthy"]:
return None
for issue in check["issues"]:
if issue == "HEARTBEAT_TIMEOUT":
self.gov.revoke_agent(agent_id, RevocationType.HEARTBEAT_TIMEOUT,
"No heartbeat received within timeout")
return "REVOKED:HEARTBEAT"
elif issue == "LOCK_EXPIRED":
self.gov.revoke_agent(agent_id, RevocationType.LOCK_EXPIRED,
"Lock expired while agent was running")
return "REVOKED:LOCK"
elif issue.startswith("STUCK_"):
self.gov.revoke_agent(agent_id, RevocationType.HEARTBEAT_TIMEOUT,
f"Agent stuck with no progress: {issue}")
return f"REVOKED:{issue}"
return None
class ComplianceWatcher:
"""
Monitors agent compliance with procedures.
- Checks required artifacts exist
- Detects forbidden actions
- Validates phase transitions
"""
def __init__(self):
self.gov = GovernanceManager()
def check_agent(self, agent_id: str) -> dict:
"""Check an agent's compliance"""
result = {
"agent_id": agent_id,
"compliant": True,
"violations": []
}
state = self.gov.get_agent_state(agent_id)
packet = self.gov.get_instruction_packet(agent_id)
if not state or not packet:
result["compliant"] = False
result["violations"].append("MISSING_STATE_OR_PACKET")
return result
# Check if EXECUTE was entered without PLAN artifact
if state.phase in [AgentPhase.EXECUTE, AgentPhase.VERIFY, AgentPhase.PACKAGE]:
if not self.gov.has_required_artifact(packet.task_id, "plan"):
result["compliant"] = False
result["violations"].append("EXECUTE_WITHOUT_PLAN_ARTIFACT")
# Check required steps from constraints
required_steps = packet.constraints.get("required_steps", [])
# This would need more sophisticated tracking in production
return result
def enforce(self, agent_id: str) -> Optional[str]:
"""Enforce compliance, return action taken"""
check = self.check_agent(agent_id)
if check["compliant"]:
return None
for violation in check["violations"]:
if violation == "EXECUTE_WITHOUT_PLAN_ARTIFACT":
self.gov.revoke_agent(agent_id, RevocationType.PROCEDURE_VIOLATION,
"Attempted EXECUTE phase without plan artifact")
return "REVOKED:NO_PLAN_ARTIFACT"
return None
class VaultLeaseWatcher:
"""
Monitors Vault token/lease validity.
- Checks token accessibility
- Confirms revocation signals
- Enforces token revocation
"""
def __init__(self):
self.gov = GovernanceManager()
self.db = self.gov.db
def set_revocation_signal(self, agent_id: str):
"""Set a signal that this agent should be revoked"""
self.db.set(f"agent:{agent_id}:revoke_signal", "1", ex=300)
def has_revocation_signal(self, agent_id: str) -> bool:
"""Check if revocation signal is set"""
return self.db.exists(f"agent:{agent_id}:revoke_signal")
def clear_revocation_signal(self, agent_id: str):
"""Clear revocation signal after enforcement"""
self.db.delete(f"agent:{agent_id}:revoke_signal")
def enforce(self, agent_id: str) -> Optional[str]:
"""Enforce revocation signal"""
if self.has_revocation_signal(agent_id):
self.gov.revoke_agent(agent_id, RevocationType.MANUAL,
"Revocation signal received")
self.clear_revocation_signal(agent_id)
return "REVOKED:SIGNAL"
return None
class GovernanceMonitorDaemon:
"""
Background daemon that runs all monitors periodically.
"""
def __init__(self, interval: int = 10):
self.interval = interval
self.gov = GovernanceManager()
self.execution_watcher = ExecutionWatcher()
self.compliance_watcher = ComplianceWatcher()
self.vault_watcher = VaultLeaseWatcher()
self.running = False
def get_active_agents(self) -> list[str]:
"""Get list of agents that need monitoring"""
# Get all agent state keys
keys = self.gov.db.keys("agent:*:state")
agents = []
for key in keys:
agent_id = key.split(":")[1]
state = self.gov.get_agent_state(agent_id)
if state and state.status == AgentStatus.RUNNING:
agents.append(agent_id)
return agents
def run_checks(self) -> list[dict]:
"""Run all monitors once"""
results = []
for agent_id in self.get_active_agents():
result = {
"agent_id": agent_id,
"timestamp": datetime.now(timezone.utc).isoformat(),
"actions": []
}
# Execution check
action = self.execution_watcher.enforce(agent_id)
if action:
result["actions"].append(action)
# Compliance check (only if not already revoked)
state = self.gov.get_agent_state(agent_id)
if state and state.status != AgentStatus.REVOKED:
action = self.compliance_watcher.enforce(agent_id)
if action:
result["actions"].append(action)
# Vault check
if state and state.status != AgentStatus.REVOKED:
action = self.vault_watcher.enforce(agent_id)
if action:
result["actions"].append(action)
if result["actions"]:
results.append(result)
return results
def run_daemon(self):
"""Run as a daemon (blocking)"""
self.running = True
print(f"[MONITOR] Starting governance monitor daemon (interval: {self.interval}s)")
while self.running:
results = self.run_checks()
for r in results:
print(f"[MONITOR] {r['agent_id']}: {r['actions']}")
time.sleep(self.interval)
def stop(self):
"""Stop the daemon"""
self.running = False
# =============================================================================
# CLI
# =============================================================================
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print("Usage: monitors.py <command>")
print("Commands:")
print(" check <agent_id> - Run all checks on an agent")
print(" daemon - Run monitor daemon")
print(" signal <agent_id> - Send revocation signal")
sys.exit(1)
cmd = sys.argv[1]
if cmd == "check":
agent_id = sys.argv[2]
exec_watch = ExecutionWatcher()
comp_watch = ComplianceWatcher()
print(f"=== Checking Agent: {agent_id} ===\n")
print("Execution Check:")
result = exec_watch.check_agent(agent_id)
print(f" Healthy: {result['healthy']}")
print(f" Issues: {result['issues']}")
print("\nCompliance Check:")
result = comp_watch.check_agent(agent_id)
print(f" Compliant: {result['compliant']}")
print(f" Violations: {result['violations']}")
elif cmd == "daemon":
daemon = GovernanceMonitorDaemon(interval=10)
try:
daemon.run_daemon()
except KeyboardInterrupt:
daemon.stop()
print("\n[MONITOR] Stopped")
elif cmd == "signal":
agent_id = sys.argv[2]
watcher = VaultLeaseWatcher()
watcher.set_revocation_signal(agent_id)
print(f"Revocation signal set for {agent_id}")
else:
print(f"Unknown command: {cmd}")
+527
View File
@@ -0,0 +1,527 @@
#!/usr/bin/env python3
"""
Promotion Engine
================
Manages agent tier progression based on compliance metrics.
Part of Phase 4: Promotion and Revocation Engine.
Promotion Rules:
- T0 → T1: 5 compliant plan-only tasks, correct scoping, clear rollbacks
- T1 → T2: N consecutive compliant sandbox runs, 2+ successful modifications
- T2 → T3: Demonstrated complexity reduction, correct escalation behavior
- T3 → T4: Rare, manual review required
"""
import json
import sqlite3
import subprocess
import sys
from dataclasses import dataclass
from datetime import datetime, timezone
from enum import Enum
from typing import Optional
# =============================================================================
# Configuration
# =============================================================================
LEDGER_DB = "/opt/agent-governance/ledger/governance.db"
# Promotion requirements per tier
PROMOTION_REQUIREMENTS = {
0: { # T0 → T1
"min_compliant_runs": 5,
"min_consecutive_compliant": 3,
"required_action_types": ["generate_plan"],
"max_violations_30d": 0,
"description": "5 compliant plan-only tasks, 3 consecutive, no violations"
},
1: { # T1 → T2
"min_compliant_runs": 10,
"min_consecutive_compliant": 5,
"required_action_types": ["ansible_check", "terraform_plan"],
"min_successful_modifications": 2,
"max_violations_30d": 0,
"description": "10 compliant runs, 5 consecutive, 2+ mods, no violations"
},
2: { # T2 → T3
"min_compliant_runs": 25,
"min_consecutive_compliant": 10,
"required_action_types": ["ansible_run", "terraform_apply"],
"min_successful_modifications": 5,
"max_violations_30d": 0,
"requires_manual_review": True,
"description": "25 compliant runs, demonstrated complexity handling"
},
3: { # T3 → T4
"min_compliant_runs": 50,
"min_consecutive_compliant": 25,
"requires_manual_review": True,
"requires_admin_approval": True,
"description": "Rare, requires admin approval and extensive track record"
}
}
class PromotionStatus(str, Enum):
ELIGIBLE = "ELIGIBLE"
NOT_ELIGIBLE = "NOT_ELIGIBLE"
PENDING_REVIEW = "PENDING_REVIEW"
APPROVED = "APPROVED"
DENIED = "DENIED"
@dataclass
class PromotionEvaluation:
agent_id: str
current_tier: int
target_tier: int
status: PromotionStatus
requirements_met: dict
requirements_missing: list
recommendation: str
evidence: dict
timestamp: str
def to_dict(self) -> dict:
return {
"agent_id": self.agent_id,
"current_tier": self.current_tier,
"target_tier": self.target_tier,
"status": self.status.value,
"requirements_met": self.requirements_met,
"requirements_missing": self.requirements_missing,
"recommendation": self.recommendation,
"evidence": self.evidence,
"timestamp": self.timestamp
}
class PromotionEngine:
"""
Evaluates and processes agent tier promotions.
"""
def __init__(self, db_path: str = LEDGER_DB):
self.db_path = db_path
def _now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _get_conn(self) -> sqlite3.Connection:
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def get_agent_metrics(self, agent_id: str) -> Optional[dict]:
"""Get current metrics for an agent"""
conn = self._get_conn()
cursor = conn.cursor()
cursor.execute("""
SELECT * FROM agent_metrics WHERE agent_id = ?
""", (agent_id,))
row = cursor.fetchone()
conn.close()
if row:
return dict(row)
return None
def get_agent_history(self, agent_id: str, days: int = 30) -> dict:
"""Get agent's action history for evaluation"""
conn = self._get_conn()
cursor = conn.cursor()
# Get total runs and successes
cursor.execute("""
SELECT
COUNT(*) as total_actions,
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful_actions,
SUM(CASE WHEN decision = 'EXECUTE' AND success = 1 THEN 1 ELSE 0 END) as successful_executions
FROM agent_actions
WHERE agent_id = ?
AND datetime(timestamp) >= datetime('now', ?)
""", (agent_id, f'-{days} days'))
action_stats = dict(cursor.fetchone())
# Get action types used
cursor.execute("""
SELECT DISTINCT action FROM agent_actions
WHERE agent_id = ?
AND datetime(timestamp) >= datetime('now', ?)
""", (agent_id, f'-{days} days'))
action_types = [row['action'] for row in cursor.fetchall()]
# Get violation count
cursor.execute("""
SELECT COUNT(*) as violation_count
FROM violations
WHERE agent_id = ?
AND datetime(timestamp) >= datetime('now', ?)
""", (agent_id, f'-{days} days'))
violations = cursor.fetchone()['violation_count']
conn.close()
return {
"total_actions": action_stats['total_actions'],
"successful_actions": action_stats['successful_actions'],
"successful_executions": action_stats['successful_executions'],
"action_types": action_types,
"violations_30d": violations
}
def evaluate_promotion(self, agent_id: str) -> PromotionEvaluation:
"""Evaluate if an agent is eligible for promotion"""
metrics = self.get_agent_metrics(agent_id)
if not metrics:
return PromotionEvaluation(
agent_id=agent_id,
current_tier=0,
target_tier=1,
status=PromotionStatus.NOT_ELIGIBLE,
requirements_met={},
requirements_missing=["Agent not found in metrics"],
recommendation="Agent has no recorded activity",
evidence={},
timestamp=self._now()
)
current_tier = metrics['current_tier']
target_tier = current_tier + 1
if target_tier > 4:
return PromotionEvaluation(
agent_id=agent_id,
current_tier=current_tier,
target_tier=current_tier,
status=PromotionStatus.NOT_ELIGIBLE,
requirements_met={},
requirements_missing=["Already at maximum tier"],
recommendation="Agent is at Tier 4 (maximum)",
evidence={"current_tier": current_tier},
timestamp=self._now()
)
requirements = PROMOTION_REQUIREMENTS.get(current_tier, {})
history = self.get_agent_history(agent_id)
# Evaluate each requirement
met = {}
missing = []
# Check compliant runs
min_runs = requirements.get('min_compliant_runs', 0)
if metrics['compliant_runs'] >= min_runs:
met['compliant_runs'] = f"{metrics['compliant_runs']} >= {min_runs}"
else:
missing.append(f"Need {min_runs - metrics['compliant_runs']} more compliant runs")
# Check consecutive compliant
min_consecutive = requirements.get('min_consecutive_compliant', 0)
if metrics['consecutive_compliant'] >= min_consecutive:
met['consecutive_compliant'] = f"{metrics['consecutive_compliant']} >= {min_consecutive}"
else:
missing.append(f"Need {min_consecutive - metrics['consecutive_compliant']} more consecutive compliant runs")
# Check violations
max_violations = requirements.get('max_violations_30d', 0)
if history['violations_30d'] <= max_violations:
met['violations_30d'] = f"{history['violations_30d']} <= {max_violations}"
else:
missing.append(f"Too many violations ({history['violations_30d']} > {max_violations})")
# Check action types
required_types = requirements.get('required_action_types', [])
for action_type in required_types:
if action_type in history['action_types']:
met[f'action_{action_type}'] = f"Has performed {action_type}"
else:
missing.append(f"Needs to perform action type: {action_type}")
# Check successful modifications (for T1+)
min_mods = requirements.get('min_successful_modifications', 0)
if min_mods > 0:
if history['successful_executions'] >= min_mods:
met['modifications'] = f"{history['successful_executions']} >= {min_mods}"
else:
missing.append(f"Need {min_mods - history['successful_executions']} more successful modifications")
# Determine status
if len(missing) == 0:
if requirements.get('requires_manual_review', False):
status = PromotionStatus.PENDING_REVIEW
recommendation = "All automatic criteria met. Manual review required."
else:
status = PromotionStatus.ELIGIBLE
recommendation = "Agent meets all promotion criteria"
else:
status = PromotionStatus.NOT_ELIGIBLE
recommendation = f"Agent needs to complete {len(missing)} requirements"
return PromotionEvaluation(
agent_id=agent_id,
current_tier=current_tier,
target_tier=target_tier,
status=status,
requirements_met=met,
requirements_missing=missing,
recommendation=recommendation,
evidence={
"metrics": metrics,
"history": history,
"requirements": requirements
},
timestamp=self._now()
)
def process_promotion(self, agent_id: str, approved_by: str,
rationale: str = None) -> tuple[bool, str]:
"""Process an approved promotion"""
evaluation = self.evaluate_promotion(agent_id)
if evaluation.status not in [PromotionStatus.ELIGIBLE, PromotionStatus.PENDING_REVIEW]:
return False, f"Agent not eligible: {evaluation.recommendation}"
conn = self._get_conn()
cursor = conn.cursor()
try:
# Update agent tier in metrics
cursor.execute("""
UPDATE agent_metrics
SET current_tier = ?,
updated_at = ?
WHERE agent_id = ?
""", (evaluation.target_tier, self._now(), agent_id))
# Record promotion
cursor.execute("""
INSERT INTO promotions (timestamp, agent_id, from_tier, to_tier,
approved_by, rationale, evidence)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (
self._now(),
agent_id,
evaluation.current_tier,
evaluation.target_tier,
approved_by,
rationale or evaluation.recommendation,
json.dumps(evaluation.evidence)
))
conn.commit()
# Update Vault policy
self._update_vault_policy(agent_id, evaluation.target_tier)
return True, f"Promoted {agent_id} from Tier {evaluation.current_tier} to Tier {evaluation.target_tier}"
except Exception as e:
conn.rollback()
return False, f"Promotion failed: {str(e)}"
finally:
conn.close()
def _update_vault_policy(self, agent_id: str, new_tier: int):
"""Update agent's Vault policy to new tier"""
# In production, this would update the AppRole's token_policies
# For now, just log the action
print(f"[VAULT] Would update {agent_id} to tier{new_tier}-agent role")
# Example Vault API call (commented out for safety):
# curl -sk -X POST \
# -H "X-Vault-Token: $TOKEN" \
# -d '{"token_policies": ["t{new_tier}-observer"]}' \
# https://127.0.0.1:8200/v1/auth/approle/role/tier{new_tier}-agent
def get_promotion_history(self, agent_id: str = None, limit: int = 20) -> list:
"""Get promotion history"""
conn = self._get_conn()
cursor = conn.cursor()
if agent_id:
cursor.execute("""
SELECT * FROM promotions
WHERE agent_id = ?
ORDER BY timestamp DESC
LIMIT ?
""", (agent_id, limit))
else:
cursor.execute("""
SELECT * FROM promotions
ORDER BY timestamp DESC
LIMIT ?
""", (limit,))
rows = [dict(row) for row in cursor.fetchall()]
conn.close()
return rows
def get_all_eligible_agents(self) -> list[PromotionEvaluation]:
"""Find all agents eligible for promotion"""
conn = self._get_conn()
cursor = conn.cursor()
cursor.execute("SELECT agent_id FROM agent_metrics")
agents = [row['agent_id'] for row in cursor.fetchall()]
conn.close()
eligible = []
for agent_id in agents:
eval = self.evaluate_promotion(agent_id)
if eval.status in [PromotionStatus.ELIGIBLE, PromotionStatus.PENDING_REVIEW]:
eligible.append(eval)
return eligible
# =============================================================================
# CLI
# =============================================================================
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Promotion Engine")
subparsers = parser.add_subparsers(dest="command", required=True)
# Evaluate command
eval_parser = subparsers.add_parser("evaluate", help="Evaluate agent for promotion")
eval_parser.add_argument("agent_id", help="Agent ID to evaluate")
eval_parser.add_argument("--json", action="store_true", help="Output JSON")
# Promote command
promote_parser = subparsers.add_parser("promote", help="Process promotion")
promote_parser.add_argument("agent_id", help="Agent ID to promote")
promote_parser.add_argument("--approved-by", required=True, help="Approver name")
promote_parser.add_argument("--rationale", help="Promotion rationale")
# List eligible command
eligible_parser = subparsers.add_parser("eligible", help="List eligible agents")
eligible_parser.add_argument("--json", action="store_true", help="Output JSON")
# History command
history_parser = subparsers.add_parser("history", help="View promotion history")
history_parser.add_argument("--agent-id", help="Filter by agent")
history_parser.add_argument("--limit", type=int, default=20)
history_parser.add_argument("--json", action="store_true", help="Output JSON")
# Requirements command
req_parser = subparsers.add_parser("requirements", help="Show promotion requirements")
req_parser.add_argument("--tier", type=int, help="Specific tier")
args = parser.parse_args()
engine = PromotionEngine()
if args.command == "evaluate":
result = engine.evaluate_promotion(args.agent_id)
if args.json:
print(json.dumps(result.to_dict(), indent=2))
else:
print("\n" + "=" * 60)
print("PROMOTION EVALUATION")
print("=" * 60)
print(f"Agent: {result.agent_id}")
print(f"Current Tier: {result.current_tier}")
print(f"Target Tier: {result.target_tier}")
print(f"Status: {result.status.value}")
print()
print("Requirements Met:")
for key, value in result.requirements_met.items():
print(f" [OK] {key}: {value}")
if result.requirements_missing:
print("\nRequirements Missing:")
for req in result.requirements_missing:
print(f" [X] {req}")
print(f"\nRecommendation: {result.recommendation}")
print("=" * 60)
elif args.command == "promote":
success, message = engine.process_promotion(
args.agent_id,
args.approved_by,
args.rationale
)
if success:
print(f"[OK] {message}")
else:
print(f"[FAILED] {message}")
sys.exit(1)
elif args.command == "eligible":
eligible = engine.get_all_eligible_agents()
if args.json:
print(json.dumps([e.to_dict() for e in eligible], indent=2))
else:
print("\n" + "=" * 60)
print("AGENTS ELIGIBLE FOR PROMOTION")
print("=" * 60)
if not eligible:
print("No agents currently eligible for promotion")
else:
for e in eligible:
status_icon = "[READY]" if e.status == PromotionStatus.ELIGIBLE else "[REVIEW]"
print(f"\n{status_icon} {e.agent_id}")
print(f" Tier {e.current_tier} → Tier {e.target_tier}")
print(f" {e.recommendation}")
print("=" * 60)
elif args.command == "history":
history = engine.get_promotion_history(args.agent_id, args.limit)
if args.json:
print(json.dumps(history, indent=2))
else:
print("\n" + "=" * 60)
print("PROMOTION HISTORY")
print("=" * 60)
if not history:
print("No promotion history found")
else:
for p in history:
print(f"\n{p['timestamp']}")
print(f" {p['agent_id']}: Tier {p['from_tier']} → Tier {p['to_tier']}")
print(f" Approved by: {p['approved_by']}")
if p['rationale']:
print(f" Rationale: {p['rationale']}")
print("=" * 60)
elif args.command == "requirements":
print("\n" + "=" * 60)
print("PROMOTION REQUIREMENTS")
print("=" * 60)
tiers = [args.tier] if args.tier is not None else range(4)
for tier in tiers:
req = PROMOTION_REQUIREMENTS.get(tier, {})
print(f"\nTier {tier} → Tier {tier + 1}:")
print(f" {req.get('description', 'No description')}")
print(f" - Min compliant runs: {req.get('min_compliant_runs', 'N/A')}")
print(f" - Min consecutive: {req.get('min_consecutive_compliant', 'N/A')}")
print(f" - Max violations (30d): {req.get('max_violations_30d', 'N/A')}")
if req.get('requires_manual_review'):
print(f" - Requires manual review: Yes")
if req.get('requires_admin_approval'):
print(f" - Requires admin approval: Yes")
print("=" * 60)
+637
View File
@@ -0,0 +1,637 @@
#!/usr/bin/env python3
"""
Revocation Engine
=================
Real-time violation detection and immediate credential revocation.
Part of Phase 4: Promotion and Revocation Engine.
Immediate Revocation Events:
- Resource created outside approved pool
- Terraform apply without stored plan
- Ansible run without check-mode (no waiver)
- Prod access without gate approval
- Unrecorded root session
- Direct baseline mutation
- Error budget exceeded
- Procedure violations
"""
import json
import sqlite3
import subprocess
import sys
import redis
from dataclasses import dataclass
from datetime import datetime, timezone
from enum import Enum
from typing import Optional
# =============================================================================
# Configuration
# =============================================================================
LEDGER_DB = "/opt/agent-governance/ledger/governance.db"
class ViolationType(str, Enum):
# Critical - Immediate revocation
UNAUTHORIZED_POOL = "UNAUTHORIZED_POOL"
APPLY_WITHOUT_PLAN = "APPLY_WITHOUT_PLAN"
RUN_WITHOUT_CHECK = "RUN_WITHOUT_CHECK"
UNAUTHORIZED_PROD = "UNAUTHORIZED_PROD"
UNRECORDED_ROOT = "UNRECORDED_ROOT"
BASELINE_MUTATION = "BASELINE_MUTATION"
# High - Immediate revocation
ERROR_BUDGET_EXCEEDED = "ERROR_BUDGET_EXCEEDED"
PROCEDURE_VIOLATION = "PROCEDURE_VIOLATION"
HEARTBEAT_TIMEOUT = "HEARTBEAT_TIMEOUT"
LOCK_EXPIRED = "LOCK_EXPIRED"
# Medium - Warning then revocation
SCOPE_VIOLATION = "SCOPE_VIOLATION"
FORBIDDEN_ACTION = "FORBIDDEN_ACTION"
# Low - Warning only
CONFIDENCE_BELOW_THRESHOLD = "CONFIDENCE_BELOW_THRESHOLD"
MISSING_ARTIFACT = "MISSING_ARTIFACT"
class Severity(str, Enum):
CRITICAL = "critical" # Immediate revocation, alert
HIGH = "high" # Immediate revocation
MEDIUM = "medium" # Warning, second offense = revoke
LOW = "low" # Warning only
# Violation severity mapping
VIOLATION_SEVERITY = {
ViolationType.UNAUTHORIZED_POOL: Severity.CRITICAL,
ViolationType.APPLY_WITHOUT_PLAN: Severity.CRITICAL,
ViolationType.RUN_WITHOUT_CHECK: Severity.CRITICAL,
ViolationType.UNAUTHORIZED_PROD: Severity.CRITICAL,
ViolationType.UNRECORDED_ROOT: Severity.CRITICAL,
ViolationType.BASELINE_MUTATION: Severity.CRITICAL,
ViolationType.ERROR_BUDGET_EXCEEDED: Severity.HIGH,
ViolationType.PROCEDURE_VIOLATION: Severity.HIGH,
ViolationType.HEARTBEAT_TIMEOUT: Severity.HIGH,
ViolationType.LOCK_EXPIRED: Severity.HIGH,
ViolationType.SCOPE_VIOLATION: Severity.MEDIUM,
ViolationType.FORBIDDEN_ACTION: Severity.MEDIUM,
ViolationType.CONFIDENCE_BELOW_THRESHOLD: Severity.LOW,
ViolationType.MISSING_ARTIFACT: Severity.LOW,
}
@dataclass
class Violation:
agent_id: str
violation_type: ViolationType
severity: Severity
description: str
triggering_action: str
evidence: dict
timestamp: str
remediation: str = ""
def to_dict(self) -> dict:
return {
"agent_id": self.agent_id,
"violation_type": self.violation_type.value,
"severity": self.severity.value,
"description": self.description,
"triggering_action": self.triggering_action,
"evidence": self.evidence,
"timestamp": self.timestamp,
"remediation": self.remediation
}
@dataclass
class RevocationResult:
agent_id: str
success: bool
action_taken: str
violation: Violation
vault_revoked: bool
dragonfly_revoked: bool
timestamp: str
def to_dict(self) -> dict:
return {
"agent_id": self.agent_id,
"success": self.success,
"action_taken": self.action_taken,
"violation": self.violation.to_dict(),
"vault_revoked": self.vault_revoked,
"dragonfly_revoked": self.dragonfly_revoked,
"timestamp": self.timestamp
}
class RevocationEngine:
"""
Detects violations and revokes agent access.
"""
def __init__(self, db_path: str = LEDGER_DB):
self.db_path = db_path
self.vault_token = self._get_vault_token()
self.redis = self._get_redis()
def _now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _get_vault_token(self) -> str:
try:
with open("/opt/vault/init-keys.json") as f:
return json.load(f)["root_token"]
except:
return ""
def _get_redis(self) -> Optional[redis.Redis]:
try:
# Get credentials from Vault
result = subprocess.run([
"curl", "-sk",
"-H", f"X-Vault-Token: {self.vault_token}",
"https://127.0.0.1:8200/v1/secret/data/services/dragonfly"
], capture_output=True, text=True)
creds = json.loads(result.stdout)["data"]["data"]
return redis.Redis(
host=creds["host"],
port=int(creds["port"]),
password=creds["password"],
decode_responses=True
)
except:
return None
def _get_conn(self) -> sqlite3.Connection:
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def record_violation(self, violation: Violation) -> int:
"""Record a violation in the ledger"""
conn = self._get_conn()
cursor = conn.cursor()
cursor.execute("""
INSERT INTO violations
(timestamp, agent_id, violation_type, severity, description,
triggering_action, evidence, remediation)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
violation.timestamp,
violation.agent_id,
violation.violation_type.value,
violation.severity.value,
violation.description,
violation.triggering_action,
json.dumps(violation.evidence),
violation.remediation
))
violation_id = cursor.lastrowid
# Update agent metrics
cursor.execute("""
UPDATE agent_metrics
SET consecutive_compliant = 0,
last_violation_at = ?,
promotion_eligible = 0,
updated_at = ?
WHERE agent_id = ?
""", (violation.timestamp, violation.timestamp, violation.agent_id))
conn.commit()
conn.close()
return violation_id
def revoke_vault_token(self, agent_id: str) -> bool:
"""Revoke agent's Vault token"""
# In production, this would look up the agent's token accessor
# and revoke it via Vault API
# For now, set a revocation signal that the agent must respect
try:
result = subprocess.run([
"curl", "-sk", "-X", "POST",
"-H", f"X-Vault-Token: {self.vault_token}",
"-d", json.dumps({"revoked_at": self._now(), "agent_id": agent_id}),
f"https://127.0.0.1:8200/v1/secret/data/revocations/{agent_id}"
], capture_output=True, text=True)
return "errors" not in result.stdout
except:
return False
def revoke_dragonfly_access(self, agent_id: str) -> bool:
"""Revoke agent's access in DragonflyDB"""
if not self.redis:
return False
try:
# Set revocation signal
self.redis.set(f"agent:{agent_id}:revoke_signal", "1", ex=86400)
# Update agent state to REVOKED
state_key = f"agent:{agent_id}:state"
state_data = self.redis.get(state_key)
if state_data:
state = json.loads(state_data)
state["status"] = "REVOKED"
state["revoked_at"] = self._now()
self.redis.set(state_key, json.dumps(state))
# Release any locks
self.redis.delete(f"agent:{agent_id}:lock")
# Add to revocation ledger
revocation_event = {
"agent_id": agent_id,
"revoked_at": self._now(),
"reason": "VIOLATION"
}
self.redis.rpush("revocations:ledger", json.dumps(revocation_event))
return True
except Exception as e:
print(f"[ERROR] Failed to revoke DragonflyDB access: {e}")
return False
def send_alert(self, violation: Violation):
"""Send alert for critical violations"""
# In production, this would integrate with PagerDuty, Slack, etc.
print(f"\n{'!'*60}")
print(f"CRITICAL ALERT: {violation.violation_type.value}")
print(f"{'!'*60}")
print(f"Agent: {violation.agent_id}")
print(f"Description: {violation.description}")
print(f"Severity: {violation.severity.value}")
print(f"Time: {violation.timestamp}")
print(f"{'!'*60}\n")
# Store alert for dashboard
if self.redis:
alert = {
"type": "VIOLATION",
"violation_type": violation.violation_type.value,
"agent_id": violation.agent_id,
"severity": violation.severity.value,
"message": violation.description,
"timestamp": violation.timestamp
}
self.redis.rpush("alerts:queue", json.dumps(alert))
self.redis.ltrim("alerts:queue", -100, -1) # Keep last 100 alerts
def process_violation(self, violation: Violation) -> RevocationResult:
"""Process a violation and take appropriate action"""
print(f"\n[VIOLATION] Processing: {violation.violation_type.value}")
print(f"[VIOLATION] Agent: {violation.agent_id}")
print(f"[VIOLATION] Severity: {violation.severity.value}")
# Record the violation
violation_id = self.record_violation(violation)
# Determine action based on severity
vault_revoked = False
dragonfly_revoked = False
action_taken = "RECORDED"
if violation.severity in [Severity.CRITICAL, Severity.HIGH]:
# Immediate revocation
print(f"[REVOKE] Initiating immediate revocation for {violation.agent_id}")
vault_revoked = self.revoke_vault_token(violation.agent_id)
dragonfly_revoked = self.revoke_dragonfly_access(violation.agent_id)
action_taken = "REVOKED"
if violation.severity == Severity.CRITICAL:
self.send_alert(violation)
elif violation.severity == Severity.MEDIUM:
# Check for prior warnings
conn = self._get_conn()
cursor = conn.cursor()
cursor.execute("""
SELECT COUNT(*) as count FROM violations
WHERE agent_id = ?
AND severity = 'medium'
AND datetime(timestamp) >= datetime('now', '-7 days')
""", (violation.agent_id,))
prior_warnings = cursor.fetchone()['count']
conn.close()
if prior_warnings >= 2: # This is the 3rd+ medium violation
print(f"[REVOKE] Multiple medium violations, revoking {violation.agent_id}")
vault_revoked = self.revoke_vault_token(violation.agent_id)
dragonfly_revoked = self.revoke_dragonfly_access(violation.agent_id)
action_taken = "REVOKED"
else:
print(f"[WARNING] Warning issued to {violation.agent_id}")
action_taken = "WARNING"
else: # LOW severity
print(f"[WARNING] Low severity violation recorded for {violation.agent_id}")
action_taken = "WARNING"
return RevocationResult(
agent_id=violation.agent_id,
success=True,
action_taken=action_taken,
violation=violation,
vault_revoked=vault_revoked,
dragonfly_revoked=dragonfly_revoked,
timestamp=self._now()
)
def create_violation(
self,
agent_id: str,
violation_type: ViolationType,
description: str,
triggering_action: str = "",
evidence: dict = None,
remediation: str = ""
) -> Violation:
"""Helper to create a violation object"""
return Violation(
agent_id=agent_id,
violation_type=violation_type,
severity=VIOLATION_SEVERITY[violation_type],
description=description,
triggering_action=triggering_action,
evidence=evidence or {},
timestamp=self._now(),
remediation=remediation
)
def get_violations(self, agent_id: str = None, severity: str = None,
limit: int = 50) -> list:
"""Get violation history"""
conn = self._get_conn()
cursor = conn.cursor()
query = "SELECT * FROM violations WHERE 1=1"
params = []
if agent_id:
query += " AND agent_id = ?"
params.append(agent_id)
if severity:
query += " AND severity = ?"
params.append(severity)
query += " ORDER BY timestamp DESC LIMIT ?"
params.append(limit)
cursor.execute(query, params)
rows = [dict(row) for row in cursor.fetchall()]
conn.close()
return rows
def get_active_revocations(self) -> list:
"""Get currently revoked agents from DragonflyDB"""
if not self.redis:
return []
revoked = []
keys = self.redis.keys("agent:*:state")
for key in keys:
data = self.redis.get(key)
if data:
state = json.loads(data)
if state.get("status") == "REVOKED":
revoked.append({
"agent_id": state.get("agent_id"),
"revoked_at": state.get("revoked_at"),
"notes": state.get("notes", "")
})
return revoked
def acknowledge_violation(self, violation_id: int, acknowledged_by: str) -> bool:
"""Acknowledge a violation (for remediation tracking)"""
conn = self._get_conn()
cursor = conn.cursor()
cursor.execute("""
UPDATE violations
SET acknowledged = 1,
acknowledged_by = ?
WHERE id = ?
""", (acknowledged_by, violation_id))
success = cursor.rowcount > 0
conn.commit()
conn.close()
return success
# =============================================================================
# Violation Detectors
# =============================================================================
class ViolationDetector:
"""
Detects violations in real-time.
"""
def __init__(self):
self.engine = RevocationEngine()
def check_plan_artifact(self, agent_id: str, action: str, artifact_exists: bool) -> Optional[Violation]:
"""Check if apply/run has a corresponding plan artifact"""
if action in ["terraform_apply", "ansible_run"] and not artifact_exists:
return self.engine.create_violation(
agent_id=agent_id,
violation_type=ViolationType.APPLY_WITHOUT_PLAN,
description=f"Attempted {action} without required plan artifact",
triggering_action=action,
remediation="Always run plan/check before apply/run"
)
return None
def check_pool_authorization(self, agent_id: str, agent_tier: int,
target_pool: str) -> Optional[Violation]:
"""Check if agent is authorized for target pool"""
forbidden_pools = {
0: ["pve-sandbox", "pve-staging", "pve-prod"],
1: ["pve-staging", "pve-prod"],
2: ["pve-prod"],
3: [],
4: []
}
if target_pool in forbidden_pools.get(agent_tier, []):
return self.engine.create_violation(
agent_id=agent_id,
violation_type=ViolationType.UNAUTHORIZED_POOL,
description=f"Tier {agent_tier} agent accessed forbidden pool: {target_pool}",
evidence={"agent_tier": agent_tier, "target_pool": target_pool},
remediation="Request promotion or use authorized pools only"
)
return None
def check_production_access(self, agent_id: str, target: str,
has_approval: bool) -> Optional[Violation]:
"""Check if production access has gate approval"""
if "prod" in target.lower() and not has_approval:
return self.engine.create_violation(
agent_id=agent_id,
violation_type=ViolationType.UNAUTHORIZED_PROD,
description=f"Production access attempted without approval: {target}",
evidence={"target": target, "approval": False},
remediation="Request production access through gate approval process"
)
return None
# =============================================================================
# CLI
# =============================================================================
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Revocation Engine")
subparsers = parser.add_subparsers(dest="command", required=True)
# Report violation command
report_parser = subparsers.add_parser("report", help="Report a violation")
report_parser.add_argument("agent_id", help="Agent ID")
report_parser.add_argument("--type", required=True,
choices=[v.value for v in ViolationType],
help="Violation type")
report_parser.add_argument("--description", required=True, help="Description")
report_parser.add_argument("--action", default="", help="Triggering action")
# List violations command
list_parser = subparsers.add_parser("list", help="List violations")
list_parser.add_argument("--agent-id", help="Filter by agent")
list_parser.add_argument("--severity", choices=["critical", "high", "medium", "low"])
list_parser.add_argument("--limit", type=int, default=20)
list_parser.add_argument("--json", action="store_true")
# List revoked command
revoked_parser = subparsers.add_parser("revoked", help="List revoked agents")
revoked_parser.add_argument("--json", action="store_true")
# Acknowledge command
ack_parser = subparsers.add_parser("acknowledge", help="Acknowledge violation")
ack_parser.add_argument("violation_id", type=int)
ack_parser.add_argument("--by", required=True, help="Acknowledger name")
# Types command
types_parser = subparsers.add_parser("types", help="List violation types")
args = parser.parse_args()
engine = RevocationEngine()
if args.command == "report":
violation_type = ViolationType(args.type)
violation = engine.create_violation(
agent_id=args.agent_id,
violation_type=violation_type,
description=args.description,
triggering_action=args.action
)
result = engine.process_violation(violation)
print(f"\n{'='*60}")
print("VIOLATION PROCESSED")
print(f"{'='*60}")
print(f"Agent: {result.agent_id}")
print(f"Type: {result.violation.violation_type.value}")
print(f"Severity: {result.violation.severity.value}")
print(f"Action Taken: {result.action_taken}")
print(f"Vault Revoked: {result.vault_revoked}")
print(f"DragonflyDB Revoked: {result.dragonfly_revoked}")
print(f"{'='*60}")
elif args.command == "list":
violations = engine.get_violations(
agent_id=args.agent_id,
severity=args.severity,
limit=args.limit
)
if args.json:
print(json.dumps(violations, indent=2))
else:
print(f"\n{'='*60}")
print("VIOLATION HISTORY")
print(f"{'='*60}")
if not violations:
print("No violations found")
else:
for v in violations:
severity_icon = {
"critical": "[CRIT]",
"high": "[HIGH]",
"medium": "[MED]",
"low": "[LOW]"
}.get(v['severity'], "[?]")
ack = "[ACK]" if v['acknowledged'] else ""
print(f"\n{severity_icon} {v['violation_type']} {ack}")
print(f" Agent: {v['agent_id']}")
print(f" {v['description']}")
print(f" Time: {v['timestamp']}")
print(f"{'='*60}")
elif args.command == "revoked":
revoked = engine.get_active_revocations()
if args.json:
print(json.dumps(revoked, indent=2))
else:
print(f"\n{'='*60}")
print("CURRENTLY REVOKED AGENTS")
print(f"{'='*60}")
if not revoked:
print("No agents currently revoked")
else:
for r in revoked:
print(f"\n {r['agent_id']}")
print(f" Revoked: {r['revoked_at']}")
if r['notes']:
print(f" Notes: {r['notes']}")
print(f"{'='*60}")
elif args.command == "acknowledge":
success = engine.acknowledge_violation(args.violation_id, args.by)
if success:
print(f"[OK] Violation {args.violation_id} acknowledged by {args.by}")
else:
print(f"[FAILED] Could not acknowledge violation {args.violation_id}")
sys.exit(1)
elif args.command == "types":
print(f"\n{'='*60}")
print("VIOLATION TYPES")
print(f"{'='*60}")
for vtype in ViolationType:
severity = VIOLATION_SEVERITY[vtype]
print(f"\n {vtype.value}")
print(f" Severity: {severity.value}")
print(f"{'='*60}")