Initial commit: Agent Governance System Phase 8

Phase 8 Production Hardening with complete governance infrastructure: - Vault integration with tiered policies (T0-T4) - DragonflyDB state management - SQLite audit ledger - Pipeline DSL and templates - Promotion/revocation engine - Checkpoint system for session persistence - Health manager and circuit breaker for fault tolerance - GitHub/Slack integrations - Architectural test pipeline with bug watcher, suggestion engine, council review - Multi-agent chaos testing framework Test Results: - Governance tests: 68/68 passing - E2E workflow: 16/16 passing - Phase 2 Vault: 14/14 passing - Integration tests: 27/27 passing Coverage: 57.6% average across 12 phases Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-23 22:07:06 -05:00
commit 77655c298c
338 changed files with 63491 additions and 0 deletions
@@ -0,0 +1,576 @@
+"""
+Error Injector
+==============
+Controlled fault injection for testing the oversight system.
+
+Features:
+- Inject known errors to test watcher detection
+- Validate suggestion/council response
+- Clean up injected errors after tests
+- Safe mode to prevent production impact
+"""
+
+import json
+import os
+import hashlib
+import shutil
+from datetime import datetime, timezone
+from dataclasses import dataclass, field, asdict
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional
+import redis
+
+
+class InjectionType(str, Enum):
+    """Types of errors that can be injected"""
+    MISSING_FILE = "missing_file"
+    CORRUPTED_CONFIG = "corrupted_config"
+    INVALID_STATUS = "invalid_status"
+    DEPENDENCY_FAILURE = "dependency_failure"
+    PERMISSION_ERROR = "permission_error"
+    STATE_INCONSISTENCY = "state_inconsistency"
+    TIMEOUT_SIMULATION = "timeout_simulation"
+    SECURITY_VIOLATION = "security_violation"
+
+
+class InjectionScope(str, Enum):
+    """Scope of error injection"""
+    FILE = "file"           # Single file
+    DIRECTORY = "directory"  # Directory level
+    SERVICE = "service"      # Service/dependency
+    DATABASE = "database"    # Database entry
+    MEMORY = "memory"        # In-memory state
+
+
+@dataclass
+class Injection:
+    """Represents an error injection"""
+    id: str
+    type: InjectionType
+    scope: InjectionScope
+    target: str
+    description: str
+    phase_affected: int
+    original_state: Optional[dict] = None  # For restoration
+    injected_at: str = ""
+    cleaned_up: bool = False
+    cleanup_at: Optional[str] = None
+    detected: bool = False
+    detected_at: Optional[str] = None
+    suggestion_generated: bool = False
+    council_reviewed: bool = False
+
+    def __post_init__(self):
+        if not self.injected_at:
+            self.injected_at = datetime.now(timezone.utc).isoformat()
+        if not self.id:
+            self.id = f"inj-{hashlib.sha256(f'{self.type}{self.target}{self.injected_at}'.encode()).hexdigest()[:12]}"
+
+
+@dataclass
+class InjectionResult:
+    """Result of an injection test"""
+    injection_id: str
+    detected_by_watcher: bool
+    detection_time_ms: int
+    suggestion_quality: str  # "accurate", "partial", "missed", "n/a"
+    council_decision: str    # decision type or "n/a"
+    false_positives: int
+    test_passed: bool
+    notes: str = ""
+
+
+class ErrorInjector:
+    """
+    Controlled error injection for testing oversight layers.
+
+    Safety features:
+    - All injections are tracked and reversible
+    - Safe mode prevents production impact
+    - Automatic cleanup after tests
+    """
+
+    # Predefined injection scenarios
+    SCENARIOS = {
+        "missing_config": {
+            "type": InjectionType.MISSING_FILE,
+            "scope": InjectionScope.FILE,
+            "target": "agents/tier0-agent/config/agent.json",
+            "description": "Remove agent config to test missing file detection",
+            "phase": 5
+        },
+        "corrupted_status": {
+            "type": InjectionType.INVALID_STATUS,
+            "scope": InjectionScope.FILE,
+            "target": "checkpoint/STATUS.md",
+            "description": "Corrupt STATUS.md to test status validation",
+            "phase": 5
+        },
+        "stale_checkpoint": {
+            "type": InjectionType.STATE_INCONSISTENCY,
+            "scope": InjectionScope.DIRECTORY,
+            "target": "checkpoint",
+            "description": "Create very old checkpoint to test staleness detection",
+            "phase": 5
+        },
+        "redis_key_missing": {
+            "type": InjectionType.DEPENDENCY_FAILURE,
+            "scope": InjectionScope.DATABASE,
+            "target": "oversight:watcher",
+            "description": "Delete Redis key to test dependency detection",
+            "phase": 8
+        },
+        "violation_unacked": {
+            "type": InjectionType.SECURITY_VIOLATION,
+            "scope": InjectionScope.DATABASE,
+            "target": "violations",
+            "description": "Insert unacknowledged critical violation",
+            "phase": 4
+        },
+        "blocked_directory": {
+            "type": InjectionType.STATE_INCONSISTENCY,
+            "scope": InjectionScope.DIRECTORY,
+            "target": "preflight/",
+            "description": "Mark directory as BLOCKED in STATUS.md",
+            "phase": 3
+        }
+    }
+
+    def __init__(self, base_path: str = "/opt/agent-governance", safe_mode: bool = True):
+        self.base_path = Path(base_path)
+        self.safe_mode = safe_mode
+        self.injections: list[Injection] = []
+        self.backups: dict[str, Any] = {}
+        self._redis: Optional[redis.Redis] = None
+        self._setup_redis()
+
+    def _setup_redis(self):
+        """Connect to DragonflyDB"""
+        try:
+            self._redis = redis.Redis(
+                host='127.0.0.1',
+                port=6379,
+                password='governance2026',
+                decode_responses=True
+            )
+            self._redis.ping()
+        except Exception:
+            self._redis = None
+
+    def _now(self) -> str:
+        return datetime.now(timezone.utc).isoformat()
+
+    def inject(self, scenario_name: str) -> Optional[Injection]:
+        """Inject an error using a predefined scenario"""
+        scenario = self.SCENARIOS.get(scenario_name)
+        if not scenario:
+            return None
+
+        return self.inject_custom(
+            injection_type=scenario["type"],
+            scope=scenario["scope"],
+            target=scenario["target"],
+            description=scenario["description"],
+            phase=scenario["phase"]
+        )
+
+    def inject_custom(
+        self,
+        injection_type: InjectionType,
+        scope: InjectionScope,
+        target: str,
+        description: str,
+        phase: int
+    ) -> Injection:
+        """Inject a custom error"""
+        injection = Injection(
+            id="",
+            type=injection_type,
+            scope=scope,
+            target=target,
+            description=description,
+            phase_affected=phase
+        )
+
+        # Backup original state
+        injection.original_state = self._backup_state(scope, target)
+
+        # Perform injection
+        if not self.safe_mode:
+            self._perform_injection(injection)
+        else:
+            # In safe mode, just simulate
+            injection.original_state = {"simulated": True}
+
+        self.injections.append(injection)
+        self._persist_injection(injection)
+
+        return injection
+
+    def _backup_state(self, scope: InjectionScope, target: str) -> dict:
+        """Backup state before injection"""
+        backup = {"scope": scope.value, "target": target}
+
+        if scope == InjectionScope.FILE:
+            file_path = self.base_path / target
+            if file_path.exists():
+                backup["content"] = file_path.read_text()
+                backup["exists"] = True
+            else:
+                backup["exists"] = False
+
+        elif scope == InjectionScope.DIRECTORY:
+            dir_path = self.base_path / target
+            status_file = dir_path / "STATUS.md"
+            if status_file.exists():
+                backup["status_content"] = status_file.read_text()
+
+        elif scope == InjectionScope.DATABASE and self._redis:
+            if ":" in target:
+                # Redis key
+                backup["redis_value"] = self._redis.get(target)
+                backup["redis_type"] = self._redis.type(target)
+
+        return backup
+
+    def _perform_injection(self, injection: Injection):
+        """Actually perform the injection (only in non-safe mode)"""
+        if injection.scope == InjectionScope.FILE:
+            if injection.type == InjectionType.MISSING_FILE:
+                file_path = self.base_path / injection.target
+                if file_path.exists():
+                    # Rename instead of delete for safety
+                    backup_path = file_path.with_suffix(file_path.suffix + ".injection_backup")
+                    shutil.move(str(file_path), str(backup_path))
+                    self.backups[injection.id] = {"backup_path": str(backup_path)}
+
+            elif injection.type == InjectionType.CORRUPTED_CONFIG:
+                file_path = self.base_path / injection.target
+                if file_path.exists():
+                    self.backups[injection.id] = {"original": file_path.read_text()}
+                    file_path.write_text("CORRUPTED_BY_INJECTION_TEST")
+
+        elif injection.scope == InjectionScope.DIRECTORY:
+            if injection.type == InjectionType.STATE_INCONSISTENCY:
+                dir_path = self.base_path / injection.target
+                status_file = dir_path / "STATUS.md"
+                if status_file.exists():
+                    original = status_file.read_text()
+                    self.backups[injection.id] = {"original": original}
+                    # Add BLOCKED marker
+                    status_file.write_text(original.replace("IN_PROGRESS", "BLOCKED"))
+
+        elif injection.scope == InjectionScope.DATABASE and self._redis:
+            if injection.type == InjectionType.DEPENDENCY_FAILURE:
+                key = injection.target
+                original = self._redis.get(key)
+                self.backups[injection.id] = {"redis_key": key, "original": original}
+                self._redis.delete(key)
+
+            elif injection.type == InjectionType.SECURITY_VIOLATION:
+                # Insert a test violation
+                import sqlite3
+                conn = sqlite3.connect(self.base_path / "ledger" / "governance.db")
+                cursor = conn.cursor()
+                cursor.execute("""
+                    INSERT INTO violations (agent_id, violation_type, severity, description, acknowledged, timestamp)
+                    VALUES (?, ?, ?, ?, ?, ?)
+                """, (
+                    "injection-test-agent",
+                    "INJECTION_TEST",
+                    "critical",
+                    "Test violation injected by ErrorInjector",
+                    0,
+                    self._now()
+                ))
+                self.backups[injection.id] = {"violation_id": cursor.lastrowid}
+                conn.commit()
+                conn.close()
+
+    def cleanup(self, injection_id: str) -> bool:
+        """Clean up a specific injection"""
+        injection = None
+        for inj in self.injections:
+            if inj.id == injection_id:
+                injection = inj
+                break
+
+        if not injection:
+            return False
+
+        if injection.cleaned_up:
+            return True
+
+        if self.safe_mode:
+            injection.cleaned_up = True
+            injection.cleanup_at = self._now()
+            return True
+
+        # Restore original state
+        backup = self.backups.get(injection_id, {})
+
+        if injection.scope == InjectionScope.FILE:
+            if "backup_path" in backup:
+                backup_path = Path(backup["backup_path"])
+                original_path = self.base_path / injection.target
+                if backup_path.exists():
+                    shutil.move(str(backup_path), str(original_path))
+            elif "original" in backup:
+                file_path = self.base_path / injection.target
+                file_path.write_text(backup["original"])
+
+        elif injection.scope == InjectionScope.DIRECTORY:
+            if "original" in backup:
+                dir_path = self.base_path / injection.target
+                status_file = dir_path / "STATUS.md"
+                status_file.write_text(backup["original"])
+
+        elif injection.scope == InjectionScope.DATABASE:
+            if "redis_key" in backup and self._redis:
+                if backup.get("original"):
+                    self._redis.set(backup["redis_key"], backup["original"])
+
+            if "violation_id" in backup:
+                import sqlite3
+                conn = sqlite3.connect(self.base_path / "ledger" / "governance.db")
+                cursor = conn.cursor()
+                cursor.execute("DELETE FROM violations WHERE id = ?", (backup["violation_id"],))
+                conn.commit()
+                conn.close()
+
+        injection.cleaned_up = True
+        injection.cleanup_at = self._now()
+
+        return True
+
+    def cleanup_all(self) -> int:
+        """Clean up all injections"""
+        cleaned = 0
+        for injection in self.injections:
+            if not injection.cleaned_up:
+                if self.cleanup(injection.id):
+                    cleaned += 1
+        return cleaned
+
+    def _persist_injection(self, injection: Injection):
+        """Persist injection to storage"""
+        if not self._redis:
+            return
+
+        self._redis.lpush(
+            "oversight:injections",
+            json.dumps(asdict(injection))
+        )
+
+    def mark_detected(self, injection_id: str) -> bool:
+        """Mark an injection as detected by the watcher"""
+        for injection in self.injections:
+            if injection.id == injection_id:
+                injection.detected = True
+                injection.detected_at = self._now()
+                return True
+        return False
+
+    def mark_suggestion_generated(self, injection_id: str) -> bool:
+        """Mark that a suggestion was generated for this injection"""
+        for injection in self.injections:
+            if injection.id == injection_id:
+                injection.suggestion_generated = True
+                return True
+        return False
+
+    def mark_council_reviewed(self, injection_id: str) -> bool:
+        """Mark that council reviewed this injection"""
+        for injection in self.injections:
+            if injection.id == injection_id:
+                injection.council_reviewed = True
+                return True
+        return False
+
+    def get_injections(self, active_only: bool = False) -> list[Injection]:
+        """Get all injections"""
+        if active_only:
+            return [i for i in self.injections if not i.cleaned_up]
+        return self.injections
+
+    def run_scenario(self, scenario_name: str) -> InjectionResult:
+        """Run a complete injection test scenario"""
+        import time
+
+        # Inject error
+        injection = self.inject(scenario_name)
+        if not injection:
+            return InjectionResult(
+                injection_id="",
+                detected_by_watcher=False,
+                detection_time_ms=0,
+                suggestion_quality="n/a",
+                council_decision="n/a",
+                false_positives=0,
+                test_passed=False,
+                notes=f"Unknown scenario: {scenario_name}"
+            )
+
+        # Run watcher
+        from .bug_watcher import BugWindowWatcher
+
+        start_time = time.time()
+        watcher = BugWindowWatcher(str(self.base_path))
+        anomalies = watcher.scan_phase(injection.phase_affected)
+        detection_time = int((time.time() - start_time) * 1000)
+
+        # Check if our injection was detected
+        detected = False
+        for anomaly in anomalies:
+            if injection.target in anomaly.message or injection.target in anomaly.directory:
+                detected = True
+                self.mark_detected(injection.id)
+                break
+
+        # Run suggestion engine
+        suggestion_quality = "n/a"
+        if detected:
+            from .suggestion_engine import SuggestionEngine
+            engine = SuggestionEngine(str(self.base_path))
+
+            for anomaly in anomalies:
+                suggestions = engine.generate_suggestions(anomaly)
+                if suggestions:
+                    self.mark_suggestion_generated(injection.id)
+                    suggestion_quality = "accurate" if len(suggestions) > 0 else "missed"
+                    break
+
+        # Run council review
+        council_decision = "n/a"
+        if suggestion_quality == "accurate":
+            from .council import CouncilReview
+            from .suggestion_engine import SuggestionEngine, Suggestion
+
+            council = CouncilReview(str(self.base_path))
+
+            # Get a suggestion to review
+            for anomaly in anomalies:
+                suggestions = engine.generate_suggestions(anomaly)
+                for sug in suggestions[:1]:
+                    decision = council.review_suggestion(sug)
+                    self.mark_council_reviewed(injection.id)
+                    council_decision = decision.decision.value
+                    break
+                break
+
+        # Clean up
+        self.cleanup(injection.id)
+
+        # Evaluate test
+        test_passed = (
+            detected and
+            suggestion_quality in ["accurate", "partial"] and
+            council_decision != "n/a"
+        )
+
+        return InjectionResult(
+            injection_id=injection.id,
+            detected_by_watcher=detected,
+            detection_time_ms=detection_time,
+            suggestion_quality=suggestion_quality,
+            council_decision=council_decision,
+            false_positives=len([a for a in anomalies if injection.target not in a.message]),
+            test_passed=test_passed,
+            notes=f"Scenario: {scenario_name}"
+        )
+
+    def run_all_scenarios(self) -> list[InjectionResult]:
+        """Run all predefined scenarios"""
+        results = []
+        for scenario_name in self.SCENARIOS:
+            result = self.run_scenario(scenario_name)
+            results.append(result)
+        return results
+
+    def get_summary(self) -> dict:
+        """Get summary of injections"""
+        total = len(self.injections)
+        detected = sum(1 for i in self.injections if i.detected)
+        suggestion_generated = sum(1 for i in self.injections if i.suggestion_generated)
+        council_reviewed = sum(1 for i in self.injections if i.council_reviewed)
+        cleaned = sum(1 for i in self.injections if i.cleaned_up)
+
+        return {
+            "total_injections": total,
+            "detected": detected,
+            "detection_rate": f"{detected/total*100:.1f}%" if total > 0 else "0%",
+            "suggestions_generated": suggestion_generated,
+            "council_reviewed": council_reviewed,
+            "cleaned_up": cleaned,
+            "active": total - cleaned,
+            "safe_mode": self.safe_mode
+        }
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Error Injector")
+    parser.add_argument("command", choices=["inject", "cleanup", "list", "test", "test-all"])
+    parser.add_argument("--scenario", help="Scenario name")
+    parser.add_argument("--unsafe", action="store_true", help="Disable safe mode (actually modify files)")
+    parser.add_argument("--json", action="store_true")
+
+    args = parser.parse_args()
+
+    injector = ErrorInjector(safe_mode=not args.unsafe)
+
+    if args.command == "inject" and args.scenario:
+        injection = injector.inject(args.scenario)
+        if injection:
+            print(f"Injected: {injection.id}")
+            print(f"Type: {injection.type.value}")
+            print(f"Target: {injection.target}")
+            print(f"Safe mode: {injector.safe_mode}")
+        else:
+            print(f"Unknown scenario: {args.scenario}")
+            print(f"Available: {', '.join(injector.SCENARIOS.keys())}")
+
+    elif args.command == "cleanup":
+        cleaned = injector.cleanup_all()
+        print(f"Cleaned up {cleaned} injections")
+
+    elif args.command == "list":
+        injections = injector.get_injections()
+        if args.json:
+            print(json.dumps([asdict(i) for i in injections], indent=2))
+        else:
+            for i in injections:
+                status = "✅" if i.cleaned_up else "🔴"
+                print(f"{status} [{i.id}] {i.type.value}: {i.target}")
+
+    elif args.command == "test" and args.scenario:
+        print(f"\n{'='*60}")
+        print(f"INJECTION TEST: {args.scenario}")
+        print(f"{'='*60}")
+
+        result = injector.run_scenario(args.scenario)
+
+        if args.json:
+            print(json.dumps(asdict(result), indent=2))
+        else:
+            print(f"Detected: {'✅' if result.detected_by_watcher else '❌'} ({result.detection_time_ms}ms)")
+            print(f"Suggestion: {result.suggestion_quality}")
+            print(f"Council: {result.council_decision}")
+            print(f"Test: {'PASSED ✅' if result.test_passed else 'FAILED ❌'}")
+
+    elif args.command == "test-all":
+        print(f"\n{'='*60}")
+        print(f"RUNNING ALL INJECTION SCENARIOS")
+        print(f"{'='*60}")
+
+        results = injector.run_all_scenarios()
+        passed = sum(1 for r in results if r.test_passed)
+
+        for r in results:
+            icon = "✅" if r.test_passed else "❌"
+            print(f"{icon} {r.notes}: detect={r.detected_by_watcher}, suggest={r.suggestion_quality}, council={r.council_decision}")
+
+        print(f"\n{'='*60}")
+        print(f"RESULTS: {passed}/{len(results)} tests passed")