profit/agent-governance
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: Agent Governance System Phase 8

Browse Source 
Phase 8 Production Hardening with complete governance infrastructure:

- Vault integration with tiered policies (T0-T4)
- DragonflyDB state management
- SQLite audit ledger
- Pipeline DSL and templates
- Promotion/revocation engine
- Checkpoint system for session persistence
- Health manager and circuit breaker for fault tolerance
- GitHub/Slack integrations
- Architectural test pipeline with bug watcher, suggestion engine, council review
- Multi-agent chaos testing framework

Test Results:
- Governance tests: 68/68 passing
- E2E workflow: 16/16 passing
- Phase 2 Vault: 14/14 passing
- Integration tests: 27/27 passing

Coverage: 57.6% average across 12 phases

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
2026-01-23 22:07:06 -05:00
co-authored by Claude Opus 4.5
commit 77655c298c
338 changed files with 63491 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
testing/oversight/reporter.py
+455
View File
@@ -0,0 +1,455 @@
"""
Oversight Reporter
==================
Comprehensive reporting for the architectural test pipeline.
Features:
- Aggregates data from all oversight layers
- Generates detailed reports per phase
- Tracks pending actions
- Links to checkpoints/status/memory
"""
import json
from datetime import datetime, timezone
from dataclasses import dataclass, field, asdict
from pathlib import Path
from typing import Any, Optional
import redis
@dataclass
class PhaseReport:
"""Report for a single phase"""
phase_number: int
phase_name: str
status: str
coverage_percent: float
bugs_detected: int
suggestions_generated: int
council_decisions: dict
pending_actions: list[str]
critical_issues: list[str]
recommendations: list[str]
@dataclass
class OversightReport:
"""Complete oversight report"""
report_id: str
generated_at: str
summary: dict
phase_reports: list[PhaseReport]
watcher_summary: dict
suggestion_summary: dict
council_summary: dict
injection_results: list[dict]
pending_actions: list[dict]
checkpoint_link: Optional[str] = None
memory_entries: int = 0
def to_markdown(self) -> str:
"""Convert report to markdown format"""
lines = []
# Header
lines.append("# Architectural Test Pipeline Report")
lines.append(f"\n**Generated:** {self.generated_at}")
lines.append(f"**Report ID:** {self.report_id}")
if self.checkpoint_link:
lines.append(f"**Checkpoint:** {self.checkpoint_link}")
# Executive Summary
lines.append("\n## Executive Summary\n")
lines.append(f"- **Phases Validated:** {self.summary.get('phases_validated', 0)}")
lines.append(f"- **Average Coverage:** {self.summary.get('average_coverage', 0)}%")
lines.append(f"- **Total Anomalies:** {self.summary.get('total_anomalies', 0)}")
lines.append(f"- **Critical Gaps:** {len(self.summary.get('critical_gaps', []))}")
# Phase Status Matrix
lines.append("\n## Phase Status Matrix\n")
lines.append("| Phase | Name | Status | Coverage | Bugs |")
lines.append("|-------|------|--------|----------|------|")
status_icons = {
"complete": "✅",
"in_progress": "🚧",
"blocked": "❌",
"needs_review": "⚠️",
"not_started": "⬜"
}
for pr in self.phase_reports:
icon = status_icons.get(pr.status, "❓")
lines.append(f"| {pr.phase_number} | {pr.phase_name[:40]} | {icon} {pr.status} | {pr.coverage_percent:.1f}% | {pr.bugs_detected} |")
# Bug Watcher Summary
lines.append("\n## Bug Watcher Summary\n")
lines.append(f"- **Total Anomalies:** {self.watcher_summary.get('total_anomalies', 0)}")
lines.append(f"- **Unresolved:** {self.watcher_summary.get('unresolved', 0)}")
by_severity = self.watcher_summary.get('by_severity', {})
if by_severity:
lines.append("\n**By Severity:**")
for sev, count in by_severity.items():
if count > 0:
lines.append(f"- {sev}: {count}")
# Suggestion Engine Summary
lines.append("\n## Suggestion Engine Summary\n")
lines.append(f"- **Total Suggestions:** {self.suggestion_summary.get('total', 0)}")
lines.append(f"- **Pending:** {self.suggestion_summary.get('pending', 0)}")
lines.append(f"- **Auto-fixable:** {self.suggestion_summary.get('auto_fixable', 0)}")
# Council Decisions
lines.append("\n## Council Decisions\n")
lines.append(f"- **Total Decisions:** {self.council_summary.get('total_decisions', 0)}")
lines.append(f"- **Auto-Approved:** {self.council_summary.get('auto_approved', 0)}")
lines.append(f"- **Lessons Learned:** {self.council_summary.get('learning_entries', 0)}")
by_type = self.council_summary.get('by_type', {})
if by_type:
lines.append("\n**By Decision Type:**")
for dtype, count in by_type.items():
if count > 0:
lines.append(f"- {dtype}: {count}")
# Injection Test Results
if self.injection_results:
lines.append("\n## Injection Test Results\n")
lines.append("| Scenario | Detected | Suggestion | Council | Result |")
lines.append("|----------|----------|------------|---------|--------|")
for r in self.injection_results:
detected = "✅" if r.get('detected_by_watcher') else "❌"
result = "✅ PASS" if r.get('test_passed') else "❌ FAIL"
lines.append(f"| {r.get('notes', 'Unknown')[:30]} | {detected} | {r.get('suggestion_quality', 'n/a')} | {r.get('council_decision', 'n/a')} | {result} |")
# Pending Actions
if self.pending_actions:
lines.append("\n## Pending Actions\n")
for i, action in enumerate(self.pending_actions[:20], 1):
priority = action.get('priority', 'medium')
icon = {"critical": "🔴", "high": "🟠", "medium": "🟡", "low": "🔵"}.get(priority, "⚪")
lines.append(f"{i}. {icon} **{action.get('title', 'Unknown')}**")
if action.get('phase'):
lines.append(f" - Phase: {action['phase']}")
if action.get('description'):
lines.append(f" - {action['description'][:100]}")
# Critical Issues
critical = self.summary.get('critical_gaps', [])
if critical:
lines.append("\n## Critical Issues\n")
for issue in critical[:10]:
lines.append(f"- ❌ {issue}")
# Footer
lines.append("\n---")
lines.append(f"*Report generated by Architectural Test Pipeline*")
lines.append(f"*Memory entries available: {self.memory_entries}*")
return "\n".join(lines)
class OversightReporter:
"""
Generates comprehensive reports from all oversight layers.
"""
def __init__(self, base_path: str = "/opt/agent-governance"):
self.base_path = Path(base_path)
self._redis: Optional[redis.Redis] = None
self._setup_redis()
def _setup_redis(self):
"""Connect to DragonflyDB"""
try:
self._redis = redis.Redis(
host='127.0.0.1',
port=6379,
password='governance2026',
decode_responses=True
)
self._redis.ping()
except Exception:
self._redis = None
def _now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def generate_report(self, include_injections: bool = False) -> OversightReport:
"""Generate a complete oversight report"""
from .bug_watcher import BugWindowWatcher
from .suggestion_engine import SuggestionEngine
from .council import CouncilReview
from .phase_validator import PhaseValidator
from .error_injector import ErrorInjector
# Initialize components
watcher = BugWindowWatcher(str(self.base_path))
engine = SuggestionEngine(str(self.base_path))
council = CouncilReview(str(self.base_path))
validator = PhaseValidator(str(self.base_path))
# Run validation
watcher.start()
validation_results = validator.validate_all_phases()
# Gather summaries
watcher_summary = watcher.get_summary()
suggestion_summary = engine.get_summary()
council_summary = council.get_summary()
phase_summary = validator.get_summary()
# Build phase reports
phase_reports = []
for phase_num, result in validation_results.items():
# Get council decisions for this phase
phase_anomalies = watcher.scan_phase(phase_num)
decisions = {"auto_approve": 0, "human_approve": 0, "reject": 0, "defer": 0}
# Handle both enum and string status
status_val = result.status.value if hasattr(result.status, 'value') else result.status
pr = PhaseReport(
phase_number=phase_num,
phase_name=result.phase_name,
status=status_val,
coverage_percent=result.coverage_percent,
bugs_detected=result.anomalies_found,
suggestions_generated=len(phase_anomalies),
council_decisions=decisions,
pending_actions=[f"Address: {g}" for g in result.gaps[:3]],
critical_issues=[g for g in result.gaps if "critical" in g.lower() or "missing" in g.lower()][:3],
recommendations=result.recommendations
)
phase_reports.append(pr)
# Run injection tests if requested
injection_results = []
if include_injections:
injector = ErrorInjector(str(self.base_path), safe_mode=True)
for scenario in list(injector.SCENARIOS.keys())[:3]: # Limit for speed
result = injector.run_scenario(scenario)
injection_results.append(asdict(result))
# Gather pending actions
pending_actions = self._gather_pending_actions(phase_reports, watcher_summary)
# Get checkpoint link
checkpoint_link = self._get_latest_checkpoint()
# Count memory entries
memory_entries = self._count_memory_entries()
# Generate report ID
report_id = f"rpt-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
report = OversightReport(
report_id=report_id,
generated_at=self._now(),
summary=phase_summary,
phase_reports=phase_reports,
watcher_summary=watcher_summary,
suggestion_summary=suggestion_summary,
council_summary=council_summary,
injection_results=injection_results,
pending_actions=pending_actions,
checkpoint_link=checkpoint_link,
memory_entries=memory_entries
)
# Persist report
self._persist_report(report)
return report
def _gather_pending_actions(self, phase_reports: list[PhaseReport], watcher_summary: dict) -> list[dict]:
"""Gather all pending actions from various sources"""
actions = []
# From phase reports
for pr in phase_reports:
for action in pr.pending_actions:
actions.append({
"title": action,
"phase": pr.phase_number,
"priority": "high" if pr.status in ["blocked", "needs_review"] else "medium",
"source": "phase_validator"
})
for rec in pr.recommendations[:2]:
actions.append({
"title": rec,
"phase": pr.phase_number,
"priority": "medium",
"source": "recommendation"
})
# From watcher (unresolved anomalies)
unresolved = watcher_summary.get('unresolved', 0)
if unresolved > 0:
by_severity = watcher_summary.get('by_severity', {})
if by_severity.get('critical', 0) > 0:
actions.append({
"title": f"Address {by_severity['critical']} critical anomalies",
"priority": "critical",
"source": "bug_watcher"
})
if by_severity.get('high', 0) > 0:
actions.append({
"title": f"Address {by_severity['high']} high-severity anomalies",
"priority": "high",
"source": "bug_watcher"
})
# Sort by priority
priority_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
actions.sort(key=lambda x: priority_order.get(x.get('priority', 'medium'), 2))
return actions[:30] # Limit
def _get_latest_checkpoint(self) -> Optional[str]:
"""Get latest checkpoint ID"""
checkpoint_dir = self.base_path / "checkpoint" / "storage"
if not checkpoint_dir.exists():
return None
checkpoints = sorted(checkpoint_dir.glob("ckpt-*.json"), reverse=True)
if checkpoints:
try:
data = json.loads(checkpoints[0].read_text())
return data.get("checkpoint_id")
except Exception:
pass
return None
def _count_memory_entries(self) -> int:
"""Count memory entries"""
memory_dir = self.base_path / "memory" / "summaries"
if not memory_dir.exists():
return 0
return len(list(memory_dir.glob("*.json")))
def _persist_report(self, report: OversightReport):
"""Persist report to storage"""
# Save to file
reports_dir = self.base_path / "testing" / "oversight" / "reports"
reports_dir.mkdir(exist_ok=True)
report_file = reports_dir / f"{report.report_id}.json"
report_file.write_text(json.dumps(asdict(report), indent=2, default=str))
# Save markdown version
md_file = reports_dir / f"{report.report_id}.md"
md_file.write_text(report.to_markdown())
# Update Redis
if self._redis:
self._redis.set("oversight:latest_report", report.report_id)
self._redis.lpush("oversight:reports", report.report_id)
self._redis.ltrim("oversight:reports", 0, 99)
def get_latest_report(self) -> Optional[OversightReport]:
"""Get the latest report"""
reports_dir = self.base_path / "testing" / "oversight" / "reports"
if not reports_dir.exists():
return None
reports = sorted(reports_dir.glob("rpt-*.json"), reverse=True)
if reports:
try:
data = json.loads(reports[0].read_text())
# Convert phase_reports back to objects
phase_reports = [PhaseReport(**pr) for pr in data.pop('phase_reports', [])]
return OversightReport(**data, phase_reports=phase_reports)
except Exception:
pass
return None
def list_reports(self, limit: int = 20) -> list[dict]:
"""List available reports"""
reports_dir = self.base_path / "testing" / "oversight" / "reports"
if not reports_dir.exists():
return []
reports = sorted(reports_dir.glob("rpt-*.json"), reverse=True)[:limit]
result = []
for report_file in reports:
try:
data = json.loads(report_file.read_text())
result.append({
"report_id": data.get("report_id"),
"generated_at": data.get("generated_at"),
"phases_validated": data.get("summary", {}).get("phases_validated", 0),
"total_anomalies": data.get("summary", {}).get("total_anomalies", 0)
})
except Exception:
continue
return result
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Oversight Reporter")
parser.add_argument("command", choices=["generate", "latest", "list", "view"])
parser.add_argument("--inject", action="store_true", help="Include injection tests")
parser.add_argument("--markdown", action="store_true", help="Output as markdown")
parser.add_argument("--json", action="store_true")
parser.add_argument("--report-id", help="Report ID to view")
args = parser.parse_args()
reporter = OversightReporter()
if args.command == "generate":
print("Generating oversight report...")
report = reporter.generate_report(include_injections=args.inject)
if args.markdown:
print(report.to_markdown())
elif args.json:
print(json.dumps(asdict(report), indent=2, default=str))
else:
print(f"\n{'='*60}")
print(f"OVERSIGHT REPORT: {report.report_id}")
print(f"{'='*60}")
print(f"Generated: {report.generated_at}")
print(f"Phases: {report.summary.get('phases_validated', 0)}")
print(f"Coverage: {report.summary.get('average_coverage', 0)}%")
print(f"Anomalies: {report.summary.get('total_anomalies', 0)}")
print(f"\nReport saved to: testing/oversight/reports/{report.report_id}.md")
elif args.command == "latest":
report = reporter.get_latest_report()
if report:
if args.markdown:
print(report.to_markdown())
else:
print(f"Latest Report: {report.report_id}")
print(f"Generated: {report.generated_at}")
else:
print("No reports found")
elif args.command == "list":
reports = reporter.list_reports()
if args.json:
print(json.dumps(reports, indent=2))
else:
for r in reports:
print(f"[{r['report_id']}] {r['generated_at']} - {r['total_anomalies']} anomalies")
elif args.command == "view" and args.report_id:
reports_dir = Path("/opt/agent-governance/testing/oversight/reports")
report_file = reports_dir / f"{args.report_id}.md"
if report_file.exists():
print(report_file.read_text())
else:
print(f"Report not found: {args.report_id}")