profit/agent-governance
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement real auto-recovery with handoff chain

Browse Source 
Orchestrator changes:
- Add dumpAgentHandoff() to dump proposals/analysis before abort
- Add loadRecoveryContext() to load inherited context on recovery runs
- Add preseedBlackboard() to pre-seed inherited proposals
- Force-spawn GAMMA immediately on recovery runs
- Track isRecoveryRun, recoveryAttempt, inheritedContext, forceGamma

Server changes:
- Update recordConsensusFailure() to read orchestrator handoff JSON
- Add collectFromBlackboard() helper as fallback
- Update triggerAutoRecovery() with comprehensive context passing
- Store inherited_handoff reference for recovery pipelines
- Track retry_count, abort_reason, handoff_ref in recovery:* keys
- Add recovery badge and prior pipeline link in UI

Test coverage:
- test_auto_recovery.py: 6 unit tests
- test_e2e_auto_recovery.py: 5 E2E tests (handoff dump, recovery
  pipeline creation, inherited context, retry tracking, status update)

Redis tracking keys:
- handoff:{pipeline_id}:agents - orchestrator dumps proposals here
- handoff:{recovery_id}:inherited - recovery pipeline inherits from
- recovery:{pipeline_id} - retry_count, abort_reason, handoff_ref

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
2026-01-24 19:39:52 -05:00
co-authored by Claude Opus 4.5
parent a19535b580
commit c96919fe35
3 changed files with 978 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agents/multi-agent/orchestrator.ts
+280 -1
View File
@@ -16,6 +16,30 @@ import {
MetricsCollector,
} from "./coordination";
import { AgentAlpha, AgentBeta, AgentGamma } from "./agents";
import { createClient, RedisClientType } from "redis";
// Redis client for direct handoff writes
let handoffRedis: RedisClientType | null = null;
async function getHandoffRedis(): Promise<RedisClientType> {
if (handoffRedis && handoffRedis.isOpen) return handoffRedis;
const initKeys = await Bun.file("/opt/vault/init-keys.json").json();
const token = initKeys.root_token;
const result = await fetch(`https://127.0.0.1:8200/v1/secret/data/services/dragonfly`, {
headers: { "X-Vault-Token": token },
// @ts-ignore - Bun supports this
tls: { rejectUnauthorized: false }
}).then(r => r.json());
const creds = result.data.data;
handoffRedis = createClient({
url: `redis://${creds.host}:${creds.port}`,
password: creds.password,
});
await handoffRedis.connect();
return handoffRedis;
}
function now(): string {
return new Date().toISOString();
@@ -82,6 +106,12 @@ export class MultiAgentOrchestrator {
private lastProgressTime: number = 0;
private progressTimeout: number = 60000; // 60 seconds without progress = stuck
// Recovery context from prior runs
private isRecoveryRun: boolean = false;
private recoveryAttempt: number = 1;
private inheritedContext: any = null;
private forceGamma: boolean = false;
constructor(model: string = "anthropic/claude-sonnet-4") {
// Use environment variable for task ID if provided
this.taskId = process.env.TASK_ID || generateId();
@@ -176,6 +206,229 @@ export class MultiAgentOrchestrator {
}
}
// Load recovery context from prior run if this is a recovery pipeline
private async loadRecoveryContext(): Promise<void> {
if (!this.pipelineId) return;
try {
const redis = await getHandoffRedis();
const pipelineData = await redis.hGetAll(`pipeline:${this.pipelineId}`);
if (pipelineData.is_recovery === "true") {
this.isRecoveryRun = true;
this.recoveryAttempt = parseInt(pipelineData.recovery_attempt || "1");
this.forceGamma = pipelineData.force_gamma === "true";
console.log("\n" + "=".repeat(70));
console.log(`RECOVERY RUN - Attempt ${this.recoveryAttempt}`);
console.log("=".repeat(70));
// Load inherited handoff
const inheritedKey = pipelineData.inherited_handoff;
if (inheritedKey) {
const inheritedData = await redis.get(inheritedKey);
if (inheritedData) {
this.inheritedContext = JSON.parse(inheritedData);
console.log(`Loaded inherited context from: ${inheritedKey}`);
console.log(`- Prior proposals: ${this.inheritedContext.proposals?.length || 0}`);
console.log(`- Prior synthesis attempts: ${this.inheritedContext.synthesis_attempts?.length || 0}`);
console.log(`- Recovery hints:`);
this.inheritedContext.recovery_hints?.forEach((hint: string, i: number) => {
console.log(` ${i + 1}. ${hint}`);
});
}
}
// Also try loading prior_context (JSON string)
if (pipelineData.prior_context) {
const priorContext = JSON.parse(pipelineData.prior_context);
console.log(`Prior run: ${priorContext.prior_pipeline}`);
console.log(`Prior failure reason: ${priorContext.failure_reason}`);
console.log(`Prior iteration count: ${priorContext.iteration_count}`);
}
if (this.forceGamma) {
console.log("\nFORCE GAMMA MODE: GAMMA mediator will be spawned immediately");
}
console.log("=".repeat(70) + "\n");
}
} catch (e: any) {
this.log(`Warning: Could not load recovery context: ${e.message}`);
}
}
// Get inherited context for agents to use
getInheritedContext(): any {
return this.inheritedContext;
}
// Check if GAMMA should be force-spawned
shouldForceGamma(): boolean {
return this.forceGamma;
}
// Pre-seed the blackboard with inherited proposals from prior run
private async preseedBlackboard(): Promise<void> {
if (!this.inheritedContext) return;
try {
// Seed prior proposals into the blackboard
if (this.inheritedContext.proposals?.length > 0) {
for (const proposal of this.inheritedContext.proposals) {
await this.blackboard.write(
"solutions",
`inherited_${proposal.agent}_${proposal.key || 'proposal'}`,
{
...proposal.value,
_inherited: true,
_from_run: this.recoveryAttempt - 1
},
proposal.agent as any
);
}
this.log(`Seeded ${this.inheritedContext.proposals.length} proposals from prior run`);
}
// Seed prior synthesis attempts
if (this.inheritedContext.synthesis_attempts?.length > 0) {
for (const synthesis of this.inheritedContext.synthesis_attempts) {
await this.blackboard.write(
"synthesis",
`inherited_${synthesis.agent}_${synthesis.key || 'synthesis'}`,
{
...synthesis.value,
_inherited: true,
_from_run: this.recoveryAttempt - 1
},
synthesis.agent as any
);
}
this.log(`Seeded ${this.inheritedContext.synthesis_attempts.length} synthesis attempts from prior run`);
}
// Write recovery metadata to blackboard
await this.blackboard.write("problem", "recovery_context", {
is_recovery: true,
recovery_attempt: this.recoveryAttempt,
prior_pipeline: this.inheritedContext.from_pipeline,
prior_proposals_count: this.inheritedContext.proposals?.length || 0,
recovery_hints: this.inheritedContext.recovery_hints || [],
instructions: [
"This is a RECOVERY run - prior agents failed to reach consensus",
"Review the inherited proposals in the 'solutions' section",
"Look for common ground between prior proposals",
"GAMMA mediator will help resolve conflicts",
"Try to synthesize a solution that incorporates the best ideas from prior attempts"
]
}, "ALPHA");
} catch (e: any) {
this.log(`Warning: Failed to pre-seed blackboard: ${e.message}`);
}
}
// Collect and dump all agent proposals/analysis to handoff JSON
private async dumpAgentHandoff(): Promise<void> {
if (!this.pipelineId) return;
try {
const redis = await getHandoffRedis();
const handoffKey = `handoff:${this.pipelineId}:agents`;
// Collect all solutions from blackboard
const solutions = await this.blackboard.readSection("solutions");
const synthesis = await this.blackboard.readSection("synthesis");
const consensus = await this.blackboard.readSection("consensus");
const problem = await this.blackboard.readSection("problem");
// Get agent states
const agentStates = await this.stateManager.getAllStates();
// Get message history
const alphaMessages = await this.alphaBus.getMessageLog(50);
const betaMessages = await this.betaBus.getMessageLog(50);
const gammaMessages = this.gammaBus ? await this.gammaBus.getMessageLog(50) : [];
// Build structured handoff
const handoff = {
pipeline_id: this.pipelineId,
task_id: this.taskId,
dump_time: new Date().toISOString(),
iteration_count: this.iterationCount,
max_iterations: this.maxIterations,
gamma_active: this.gammaAgent !== undefined,
// Agent proposals and analysis
proposals: solutions.map(s => ({
agent: s.author,
key: s.key,
value: s.value,
version: s.version,
timestamp: s.timestamp
})),
// Synthesis attempts
synthesis_attempts: synthesis.map(s => ({
agent: s.author,
key: s.key,
value: s.value,
timestamp: s.timestamp
})),
// Consensus votes and discussions
consensus_state: consensus.map(c => ({
key: c.key,
value: c.value,
author: c.author,
timestamp: c.timestamp
})),
// Problem analysis
problem_analysis: problem.map(p => ({
key: p.key,
value: p.value,
author: p.author
})),
// Agent states at abort time
agent_states: agentStates.map(s => ({
role: s.role,
status: s.status,
phase: s.phase,
progress: s.progress,
blocked_reason: s.blocked_reason,
last_activity: s.last_activity
})),
// Recent message history for context
message_summary: {
alpha_last_messages: alphaMessages.slice(-10),
beta_last_messages: betaMessages.slice(-10),
gamma_last_messages: gammaMessages.slice(-10)
},
// Recovery hints
recovery_hints: [
`Iteration limit (${this.maxIterations}) exceeded after ${this.iterationCount} iterations`,
this.gammaAgent ? "GAMMA was active but could not resolve conflicts" : "GAMMA was not spawned",
`${solutions.length} proposals generated, ${synthesis.length} synthesis attempts`,
"Consider: simplifying objective, forcing GAMMA earlier, or increasing iteration limit"
]
};
// Store handoff JSON
await redis.set(handoffKey, JSON.stringify(handoff), { EX: 86400 }); // 24hr TTL
await redis.hSet(`pipeline:${this.pipelineId}`, "handoff_key", handoffKey);
await redis.hSet(`pipeline:${this.pipelineId}`, "handoff_time", handoff.dump_time);
this.log(`Agent handoff dumped: ${solutions.length} proposals, ${synthesis.length} synthesis attempts`);
} catch (e: any) {
this.log(`Failed to dump agent handoff: ${e.message}`);
}
}
private log(msg: string) {
const elapsed = this.startTime ? ((Date.now() - this.startTime) / 1000).toFixed(1) : "0.0";
console.log(`[${elapsed}s] [ORCHESTRATOR] ${msg}`);
@@ -193,6 +446,9 @@ export class MultiAgentOrchestrator {
console.log("Model: " + this.model);
console.log("=".repeat(70) + "\n");
// Check if this is a recovery run and load inherited context
await this.loadRecoveryContext();
this.log("Initializing coordination infrastructure...");
// Initialize shared infrastructure
@@ -210,6 +466,12 @@ export class MultiAgentOrchestrator {
this.log("Infrastructure connected");
// Pre-seed blackboard with inherited context if this is a recovery run
if (this.isRecoveryRun && this.inheritedContext) {
this.log("Pre-seeding blackboard with inherited context...");
await this.preseedBlackboard();
}
// Initialize message buses for ALPHA and BETA
this.alphaBus = new MessageBus(this.taskId, "ALPHA");
this.betaBus = new MessageBus(this.taskId, "BETA");
@@ -330,6 +592,19 @@ export class MultiAgentOrchestrator {
// Write task to blackboard
await this.blackboard.write("problem", "task_definition", task, "ALPHA");
// FORCE GAMMA: If this is a recovery run, spawn GAMMA immediately
if (this.forceGamma && !this.gammaAgent) {
this.log("FORCE GAMMA MODE: Spawning GAMMA mediator immediately for recovery");
const forceReason: SpawnCondition = {
type: "STUCK",
threshold: 0,
current_value: 1,
triggered: true,
description: "Force-spawned for recovery run"
};
await this.spawnGamma(forceReason);
}
// Track if we need to abort due to timeout/iteration limit
let abortReason: string | undefined;
@@ -371,7 +646,11 @@ export class MultiAgentOrchestrator {
clearInterval(this.monitorInterval);
}
// Revoke tokens for stuck agents
// CRITICAL: Dump all agent proposals/analysis to handoff JSON BEFORE revoking tokens
this.log("Dumping agent handoff data for recovery pipeline...");
await this.dumpAgentHandoff();
// Now revoke tokens for stuck agents
await this.revokeStuckAgentTokens();
// Get partial metrics and mark as failed