Implement auto-recovery for consensus failures
- Add iteration tracking and stuck detection to orchestrator - Add triggerAutoRecovery function for automatic pipeline respawn - Store structured failure context (proposals, conflicts, reason) - Force GAMMA agent on recovery attempts for conflict resolution - Limit auto-recovery to 3 attempts to prevent infinite loops - Add UI status badges for rebooting/aborted states - Add failure-context API endpoint for orchestrator handoff - Add test_auto_recovery.py with 6 passing tests Exit codes: 0=success, 1=error, 2=consensus failure, 3=aborted Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
c6554a8b3d
commit
a19535b580
@@ -77,6 +77,10 @@ export class MultiAgentOrchestrator {
|
||||
private startTime!: number;
|
||||
private monitorInterval?: ReturnType<typeof setInterval>;
|
||||
private errorCount: number = 0;
|
||||
private iterationCount: number = 0;
|
||||
private maxIterations: number = 10;
|
||||
private lastProgressTime: number = 0;
|
||||
private progressTimeout: number = 60000; // 60 seconds without progress = stuck
|
||||
|
||||
constructor(model: string = "anthropic/claude-sonnet-4") {
|
||||
// Use environment variable for task ID if provided
|
||||
@@ -93,6 +97,85 @@ export class MultiAgentOrchestrator {
|
||||
this.log(`ERROR [${severity}] ${errorType}: ${details}`);
|
||||
}
|
||||
|
||||
private updateProgress(): void {
|
||||
this.lastProgressTime = Date.now();
|
||||
this.iterationCount++;
|
||||
}
|
||||
|
||||
private isStuck(): boolean {
|
||||
if (this.lastProgressTime === 0) return false;
|
||||
return (Date.now() - this.lastProgressTime) > this.progressTimeout;
|
||||
}
|
||||
|
||||
private isIterationLimitExceeded(): boolean {
|
||||
return this.iterationCount >= this.maxIterations;
|
||||
}
|
||||
|
||||
// Request token revocation for stuck agents
|
||||
private async revokeStuckAgentTokens(): Promise<void> {
|
||||
if (!this.pipelineId) return;
|
||||
|
||||
try {
|
||||
await fetch("http://localhost:3000/api/pipeline/revoke", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
pipeline_id: this.pipelineId,
|
||||
reason: "iteration_timeout",
|
||||
details: `Agents stuck after ${this.iterationCount} iterations`
|
||||
})
|
||||
});
|
||||
this.log("Token revocation requested for stuck agents");
|
||||
} catch (e: any) {
|
||||
this.log(`Failed to revoke tokens: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Record structured failure context for auto-recovery
|
||||
private async recordFailureContext(
|
||||
reason: "stuck" | "iteration_limit" | "consensus_failed",
|
||||
metrics: CoordinationMetrics
|
||||
): Promise<void> {
|
||||
if (!this.pipelineId) return;
|
||||
|
||||
const failureContext = {
|
||||
pipeline_id: this.pipelineId,
|
||||
task_id: this.taskId,
|
||||
failure_reason: reason,
|
||||
failure_time: new Date().toISOString(),
|
||||
iteration_count: this.iterationCount,
|
||||
elapsed_ms: Date.now() - this.startTime,
|
||||
metrics: metrics,
|
||||
gamma_spawned: this.gammaAgent !== undefined,
|
||||
error_count: this.errorCount,
|
||||
recovery_hint: this.getRecoveryHint(reason)
|
||||
};
|
||||
|
||||
try {
|
||||
await fetch("http://localhost:3000/api/pipeline/failure-context", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(failureContext)
|
||||
});
|
||||
this.log(`Failure context recorded: ${reason}`);
|
||||
} catch (e: any) {
|
||||
this.log(`Failed to record failure context: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
private getRecoveryHint(reason: string): string {
|
||||
switch (reason) {
|
||||
case "stuck":
|
||||
return "Agents became unresponsive. Try with fresh agents or GAMMA mediator.";
|
||||
case "iteration_limit":
|
||||
return "Max iterations reached without consensus. Consider simplifying objective.";
|
||||
case "consensus_failed":
|
||||
return "Agents completed but disagreed. GAMMA mediator may help resolve conflicts.";
|
||||
default:
|
||||
return "Unknown failure. Review logs for details.";
|
||||
}
|
||||
}
|
||||
|
||||
private log(msg: string) {
|
||||
const elapsed = this.startTime ? ((Date.now() - this.startTime) / 1000).toFixed(1) : "0.0";
|
||||
console.log(`[${elapsed}s] [ORCHESTRATOR] ${msg}`);
|
||||
@@ -178,7 +261,24 @@ export class MultiAgentOrchestrator {
|
||||
this.log("GAMMA agent spawned and initialized");
|
||||
}
|
||||
|
||||
private async monitorConditions(): Promise<void> {
|
||||
private async monitorConditions(): Promise<{ abort: boolean; reason?: string }> {
|
||||
// Track progress
|
||||
this.updateProgress();
|
||||
|
||||
// Check for iteration timeout (stuck without progress)
|
||||
if (this.isStuck()) {
|
||||
this.log("TIMEOUT: No progress detected - triggering auto-recovery");
|
||||
await this.reportError("iteration_timeout", "high", "Agents stuck without progress");
|
||||
return { abort: true, reason: "stuck" };
|
||||
}
|
||||
|
||||
// Check for iteration limit exceeded
|
||||
if (this.isIterationLimitExceeded()) {
|
||||
this.log("LIMIT: Maximum iterations exceeded - triggering auto-recovery");
|
||||
await this.reportError("iteration_limit", "high", `Max iterations (${this.maxIterations}) exceeded`);
|
||||
return { abort: true, reason: "iteration_limit" };
|
||||
}
|
||||
|
||||
// Check stuck condition
|
||||
const stuckAgents = await this.stateManager.detectStuckAgents(30);
|
||||
if (stuckAgents.length > 0) {
|
||||
@@ -218,17 +318,31 @@ export class MultiAgentOrchestrator {
|
||||
// Log current state
|
||||
const states = await this.stateManager.getAllStates();
|
||||
const statesSummary = states.map(s => `${s.role}:${s.status}(${(s.progress * 100).toFixed(0)}%)`).join(", ");
|
||||
this.log(`Status: ${statesSummary} | Messages: ${metricsData.total_messages} | Conflicts: ${unresolvedConflicts}`);
|
||||
this.log(`Status: ${statesSummary} | Messages: ${metricsData.total_messages} | Conflicts: ${unresolvedConflicts} | Iter: ${this.iterationCount}`);
|
||||
|
||||
return { abort: false };
|
||||
}
|
||||
|
||||
async runTask(task: TaskDefinition): Promise<CoordinationMetrics> {
|
||||
async runTask(task: TaskDefinition): Promise<CoordinationMetrics & { abort_reason?: string }> {
|
||||
this.log(`Starting task: ${task.objective.slice(0, 60)}...`);
|
||||
this.lastProgressTime = Date.now();
|
||||
|
||||
// Write task to blackboard
|
||||
await this.blackboard.write("problem", "task_definition", task, "ALPHA");
|
||||
|
||||
// Start monitoring
|
||||
this.monitorInterval = setInterval(() => this.monitorConditions(), 2000);
|
||||
// Track if we need to abort due to timeout/iteration limit
|
||||
let abortReason: string | undefined;
|
||||
|
||||
// Start monitoring with abort detection
|
||||
this.monitorInterval = setInterval(async () => {
|
||||
const result = await this.monitorConditions();
|
||||
if (result.abort && result.reason) {
|
||||
abortReason = result.reason;
|
||||
if (this.monitorInterval) {
|
||||
clearInterval(this.monitorInterval);
|
||||
}
|
||||
}
|
||||
}, 2000);
|
||||
|
||||
// Run agents in parallel
|
||||
this.log("Launching ALPHA and BETA in parallel...");
|
||||
@@ -250,6 +364,23 @@ export class MultiAgentOrchestrator {
|
||||
timeoutPromise,
|
||||
]);
|
||||
|
||||
// Check if we were aborted during execution
|
||||
if (abortReason) {
|
||||
this.log(`Task aborted: ${abortReason}`);
|
||||
if (this.monitorInterval) {
|
||||
clearInterval(this.monitorInterval);
|
||||
}
|
||||
|
||||
// Revoke tokens for stuck agents
|
||||
await this.revokeStuckAgentTokens();
|
||||
|
||||
// Get partial metrics and mark as failed
|
||||
const partialMetrics = await this.metrics.finalize(false);
|
||||
await this.recordFailureContext(abortReason as any, partialMetrics);
|
||||
|
||||
return { ...partialMetrics, abort_reason: abortReason };
|
||||
}
|
||||
|
||||
this.log("Initial agents completed or timeout reached");
|
||||
|
||||
// Check if GAMMA needs to be spawned for success validation
|
||||
@@ -287,6 +418,11 @@ export class MultiAgentOrchestrator {
|
||||
// Finalize metrics
|
||||
const finalMetrics = await this.metrics.finalize(consensusAchieved);
|
||||
|
||||
// If consensus failed, record structured failure context
|
||||
if (!consensusAchieved) {
|
||||
await this.recordFailureContext("consensus_failed", finalMetrics);
|
||||
}
|
||||
|
||||
return finalMetrics;
|
||||
}
|
||||
|
||||
@@ -449,16 +585,31 @@ The solution should consider fault tolerance, data consistency, and cost optimiz
|
||||
analyzePerformance(metrics);
|
||||
|
||||
// Output special marker for server to parse consensus status
|
||||
// Format: ORCHESTRATION_RESULT:{"consensus":true/false,"metrics":{...}}
|
||||
// Format: ORCHESTRATION_RESULT:{"consensus":true/false,"metrics":{...},"abort_reason":...}
|
||||
console.log("\nORCHESTRATION_RESULT:" + JSON.stringify({
|
||||
consensus: metrics.final_consensus,
|
||||
task_id: metrics.task_id,
|
||||
metrics: metrics
|
||||
metrics: metrics,
|
||||
abort_reason: (metrics as any).abort_reason || null,
|
||||
requires_auto_recovery: !metrics.final_consensus || !!(metrics as any).abort_reason
|
||||
}));
|
||||
|
||||
// Exit with code 2 for consensus failure (distinct from error=1, success=0)
|
||||
if (!metrics.final_consensus) {
|
||||
console.log("\n[ORCHESTRATOR] Consensus NOT achieved - exiting with code 2");
|
||||
// Exit codes:
|
||||
// 0 = Success (consensus achieved)
|
||||
// 1 = Error (crash or exception)
|
||||
// 2 = Consensus failure (agents completed but no agreement)
|
||||
// 3 = Aborted (timeout/stuck/iteration limit - auto-recovery needed)
|
||||
if ((metrics as any).abort_reason) {
|
||||
console.log(`\n[ORCHESTRATOR] Task aborted: ${(metrics as any).abort_reason} - exiting with code 3 (auto-recovery needed)`);
|
||||
exitCode = 3;
|
||||
|
||||
const pipelineId = process.env.PIPELINE_ID;
|
||||
if (pipelineId) {
|
||||
await reportErrorToObservability(pipelineId, "orchestrator_aborted", "high",
|
||||
`Orchestration aborted: ${(metrics as any).abort_reason}. Auto-recovery required.`);
|
||||
}
|
||||
} else if (!metrics.final_consensus) {
|
||||
console.log("\n[ORCHESTRATOR] Consensus NOT achieved - exiting with code 2 (auto-recovery needed)");
|
||||
exitCode = 2;
|
||||
|
||||
// Report consensus failure to observability
|
||||
|
||||
Reference in New Issue
Block a user