Implement auto-recovery for consensus failures
- Add iteration tracking and stuck detection to orchestrator - Add triggerAutoRecovery function for automatic pipeline respawn - Store structured failure context (proposals, conflicts, reason) - Force GAMMA agent on recovery attempts for conflict resolution - Limit auto-recovery to 3 attempts to prevent infinite loops - Add UI status badges for rebooting/aborted states - Add failure-context API endpoint for orchestrator handoff - Add test_auto_recovery.py with 6 passing tests Exit codes: 0=success, 1=error, 2=consensus failure, 3=aborted Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
c6554a8b3d
commit
a19535b580
+280
-23
@@ -1778,6 +1778,106 @@ async function handleFallbackAction(
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-recovery: Spawn a new pipeline automatically on consensus failure
|
||||
async function triggerAutoRecovery(
|
||||
originalPipelineId: string,
|
||||
taskId: string,
|
||||
objective: string,
|
||||
model: string,
|
||||
timeout: number,
|
||||
failureContext: ConsensusFailureContext
|
||||
): Promise<{ success: boolean; message: string; new_pipeline_id?: string }> {
|
||||
const runNumber = failureContext.run_number;
|
||||
|
||||
// Limit auto-recovery attempts to prevent infinite loops
|
||||
const MAX_AUTO_RECOVERY = 3;
|
||||
if (runNumber >= MAX_AUTO_RECOVERY) {
|
||||
return {
|
||||
success: false,
|
||||
message: `Max auto-recovery attempts (${MAX_AUTO_RECOVERY}) reached. User intervention required.`
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Create a new recovery pipeline
|
||||
const newPipelineId = `pipeline-recovery-${Date.now().toString(36)}`;
|
||||
|
||||
// Prepare context summary for new agents
|
||||
const contextSummary = {
|
||||
prior_run: runNumber,
|
||||
prior_pipeline: originalPipelineId,
|
||||
failure_reason: failureContext.metrics?.abort_reason || "consensus_failed",
|
||||
prior_proposals: failureContext.proposals?.slice(0, 3) || [], // Top 3 proposals
|
||||
prior_conflicts: failureContext.conflict_history?.slice(-5) || [], // Last 5 conflicts
|
||||
recovery_hints: [
|
||||
"Previous agents failed to reach consensus",
|
||||
"Review prior proposals for common ground",
|
||||
"Consider a different approach if prior attempts converged on same solution"
|
||||
]
|
||||
};
|
||||
|
||||
await redis.hSet(`pipeline:${newPipelineId}`, {
|
||||
task_id: taskId,
|
||||
objective: objective,
|
||||
status: "STARTING",
|
||||
created_at: new Date().toISOString(),
|
||||
agents: JSON.stringify([]),
|
||||
parent_pipeline: originalPipelineId,
|
||||
is_recovery: "true",
|
||||
recovery_attempt: String(runNumber + 1),
|
||||
prior_context: JSON.stringify(contextSummary),
|
||||
force_gamma: "true", // Always spawn GAMMA on recovery attempts
|
||||
model: model,
|
||||
timeout: String(timeout),
|
||||
auto_continue: "true"
|
||||
});
|
||||
|
||||
// Update original pipeline
|
||||
await redis.hSet(`pipeline:${originalPipelineId}`, {
|
||||
status: "REBOOTING",
|
||||
recovery_pipeline: newPipelineId
|
||||
});
|
||||
|
||||
// Log the handoff reason to Dragonfly metrics
|
||||
await redis.hSet(`handoff:${originalPipelineId}`, {
|
||||
to_pipeline: newPipelineId,
|
||||
reason: failureContext.metrics?.abort_reason || "consensus_failed",
|
||||
handoff_time: new Date().toISOString(),
|
||||
context_size: JSON.stringify(contextSummary).length,
|
||||
proposals_passed: failureContext.proposals?.length || 0
|
||||
});
|
||||
|
||||
await appendPipelineLog(newPipelineId, "SYSTEM",
|
||||
`Recovery pipeline started (attempt ${runNumber + 1}/${MAX_AUTO_RECOVERY}). GAMMA mediator will be spawned.`, "INFO");
|
||||
|
||||
await appendPipelineLog(newPipelineId, "CONTEXT",
|
||||
`Prior run had ${failureContext.proposals?.length || 0} proposals, ` +
|
||||
`${failureContext.conflict_history?.length || 0} conflicts. Force-spawning GAMMA.`, "INFO");
|
||||
|
||||
// Trigger orchestration with GAMMA hint in objective
|
||||
triggerOrchestration(
|
||||
newPipelineId,
|
||||
taskId,
|
||||
`[RECOVERY ATTEMPT ${runNumber + 1}] [FORCE GAMMA] ${objective}`,
|
||||
model,
|
||||
timeout
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Recovery pipeline spawned (attempt ${runNumber + 1})`,
|
||||
new_pipeline_id: newPipelineId
|
||||
};
|
||||
|
||||
} catch (e: any) {
|
||||
console.error(`[AUTO-RECOVERY] Failed: ${e.message}`);
|
||||
return {
|
||||
success: false,
|
||||
message: `Auto-recovery error: ${e.message}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function generateFailureReport(pipelineId: string): Promise<any> {
|
||||
const failureContext = await getConsensusFailureContext(pipelineId);
|
||||
const failureHistory = await getFailureHistory(pipelineId);
|
||||
@@ -1963,6 +2063,10 @@ async function getActivePipelines(): Promise<any[]> {
|
||||
status: data.status,
|
||||
created_at: data.created_at,
|
||||
agents: data.agents ? JSON.parse(data.agents) : [],
|
||||
recovery_pipeline: data.recovery_pipeline || null,
|
||||
failure_reason: data.failure_reason || null,
|
||||
run_number: data.run_number ? parseInt(data.run_number) : 1,
|
||||
prior_pipeline: data.prior_pipeline || null,
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
@@ -2245,30 +2349,75 @@ async function triggerOrchestration(
|
||||
metrics: orchestrationResult?.metrics
|
||||
});
|
||||
|
||||
} else if (exitCode === 2) {
|
||||
// Consensus failure - agents completed but no agreement
|
||||
await redis.hSet(pipelineKey, "status", "CONSENSUS_FAILED");
|
||||
} else if (exitCode === 2 || exitCode === 3) {
|
||||
// Exit code 2 = Consensus failure, Exit code 3 = Aborted (timeout/stuck)
|
||||
// Both trigger AUTO-RECOVERY by spawning a new pipeline
|
||||
const failureType = exitCode === 2 ? "CONSENSUS_FAILED" : "ABORTED";
|
||||
const abortReason = orchestrationResult?.abort_reason || (exitCode === 2 ? "consensus_failed" : "unknown");
|
||||
|
||||
await redis.hSet(pipelineKey, "status", failureType);
|
||||
await redis.hSet(pipelineKey, "final_consensus", "false");
|
||||
if (orchestrationResult?.metrics) {
|
||||
await redis.hSet(pipelineKey, "final_metrics", JSON.stringify(orchestrationResult.metrics));
|
||||
}
|
||||
|
||||
await appendPipelineLog(pipelineId, "ORCHESTRATOR",
|
||||
"Orchestration completed but agents failed to reach consensus", "WARN");
|
||||
const logMessage = exitCode === 2
|
||||
? "Orchestration completed but agents failed to reach consensus"
|
||||
: `Orchestration aborted: ${abortReason}`;
|
||||
await appendPipelineLog(pipelineId, "ORCHESTRATOR", logMessage, "WARN");
|
||||
|
||||
// Record the failure context for retry/escalation
|
||||
await recordConsensusFailure(pipelineId, taskId, orchestrationResult?.metrics || {});
|
||||
// Record the failure context for the new pipeline
|
||||
const failureContext = await recordConsensusFailure(pipelineId, taskId, orchestrationResult?.metrics || {});
|
||||
|
||||
broadcastUpdate("orchestration_complete", {
|
||||
pipeline_id: pipelineId,
|
||||
status: "CONSENSUS_FAILED",
|
||||
consensus: false,
|
||||
metrics: orchestrationResult?.metrics,
|
||||
fallback_options: FALLBACK_OPTIONS,
|
||||
awaiting_user_action: true
|
||||
// Log to Dragonfly metrics for observability
|
||||
await redis.hSet(`metrics:${pipelineId}`, {
|
||||
failure_type: failureType,
|
||||
abort_reason: abortReason,
|
||||
failure_time: new Date().toISOString(),
|
||||
auto_recovery_triggered: "true"
|
||||
});
|
||||
|
||||
// Do NOT mark completed_at - pipeline awaits user decision
|
||||
// Broadcast failure status with auto-recovery notice
|
||||
broadcastUpdate("orchestration_complete", {
|
||||
pipeline_id: pipelineId,
|
||||
status: failureType,
|
||||
consensus: false,
|
||||
metrics: orchestrationResult?.metrics,
|
||||
abort_reason: abortReason,
|
||||
auto_recovery: true,
|
||||
message: "Consensus failure – pipeline rebooting automatically"
|
||||
});
|
||||
|
||||
// AUTO-RECOVERY: Spawn a new pipeline with the collected context
|
||||
await appendPipelineLog(pipelineId, "SYSTEM", "AUTO-RECOVERY: Spawning new pipeline with failure context...", "WARN");
|
||||
|
||||
const recoveryResult = await triggerAutoRecovery(pipelineId, taskId, objective, model, timeout, failureContext);
|
||||
|
||||
if (recoveryResult.success) {
|
||||
await redis.hSet(pipelineKey, "status", "REBOOTING");
|
||||
await redis.hSet(pipelineKey, "recovery_pipeline", recoveryResult.new_pipeline_id!);
|
||||
await appendPipelineLog(pipelineId, "SYSTEM",
|
||||
`Auto-recovery started: ${recoveryResult.new_pipeline_id}`, "INFO");
|
||||
|
||||
broadcastUpdate("pipeline_rebooting", {
|
||||
pipeline_id: pipelineId,
|
||||
new_pipeline_id: recoveryResult.new_pipeline_id,
|
||||
failure_reason: abortReason,
|
||||
failure_log_url: `/api/pipeline/consensus/report?pipeline_id=${pipelineId}`
|
||||
});
|
||||
} else {
|
||||
// Auto-recovery failed - fall back to user action
|
||||
await appendPipelineLog(pipelineId, "SYSTEM",
|
||||
`Auto-recovery failed: ${recoveryResult.message}. User action required.`, "ERROR");
|
||||
|
||||
broadcastUpdate("orchestration_complete", {
|
||||
pipeline_id: pipelineId,
|
||||
status: "RECOVERY_FAILED",
|
||||
consensus: false,
|
||||
fallback_options: FALLBACK_OPTIONS,
|
||||
awaiting_user_action: true
|
||||
});
|
||||
}
|
||||
|
||||
} else {
|
||||
// Error - crash or exception
|
||||
@@ -3917,6 +4066,14 @@ function renderDashboard(): string {
|
||||
.status-badge.retrying { background: rgba(88, 166, 255, 0.2); color: var(--accent-blue); }
|
||||
.status-badge.escalated { background: rgba(210, 153, 34, 0.2); color: var(--accent-yellow); }
|
||||
.status-badge.completed_no_consensus { background: rgba(63, 185, 80, 0.15); color: #8bc34a; }
|
||||
.status-badge.rebooting { background: rgba(57, 197, 207, 0.2); color: var(--accent-cyan); animation: pulse 1.5s infinite; }
|
||||
.status-badge.aborted { background: rgba(248, 81, 73, 0.3); color: var(--accent-red); border: 1px solid var(--accent-red); }
|
||||
.status-badge.recovery_failed { background: rgba(248, 81, 73, 0.4); color: #ff6b6b; border: 1px solid #ff6b6b; }
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.6; }
|
||||
}
|
||||
|
||||
/* Consensus Failure Alert */
|
||||
.consensus-failure-alert {
|
||||
@@ -5633,11 +5790,25 @@ function renderDashboard(): string {
|
||||
if (selectedPipelineId === msg.data.pipeline_id) {
|
||||
loadLogs(selectedPipelineId);
|
||||
}
|
||||
// Show notification
|
||||
showNotification('Consensus Failed', \`Pipeline \${msg.data.pipeline_id} failed to reach consensus. Action required.\`, 'warn');
|
||||
}
|
||||
|
||||
if (msg.type === 'orchestration_complete' && msg.data.consensus === false) {
|
||||
// Auto-recovery: Pipeline rebooting
|
||||
if (msg.type === 'pipeline_rebooting') {
|
||||
loadPipelines();
|
||||
if (selectedPipelineId === msg.data.pipeline_id) {
|
||||
loadLogs(selectedPipelineId);
|
||||
}
|
||||
showNotification('Pipeline Rebooting',
|
||||
\`Consensus failure – pipeline \${msg.data.new_pipeline_id} spawning automatically. <a href="#" onclick="showFailureDetails('\${msg.data.pipeline_id}'); return false;">View failure log</a>\`,
|
||||
'info');
|
||||
}
|
||||
|
||||
if (msg.type === 'orchestration_complete' && msg.data.auto_recovery) {
|
||||
loadPipelines();
|
||||
showNotification('Auto-Recovery', msg.data.message || 'Consensus failure – pipeline rebooting automatically.', 'info');
|
||||
}
|
||||
|
||||
if (msg.type === 'orchestration_complete' && msg.data.consensus === false && !msg.data.auto_recovery) {
|
||||
loadPipelines();
|
||||
showNotification('Consensus Failed', 'Agents completed but could not agree. Choose a fallback action.', 'warn');
|
||||
}
|
||||
@@ -5746,7 +5917,9 @@ function renderDashboard(): string {
|
||||
container.innerHTML = pipelinesData.map(p => {
|
||||
const isActive = p.pipeline_id === selectedPipelineId;
|
||||
const agents = p.agents || [];
|
||||
const isConsensusFailed = p.status === 'CONSENSUS_FAILED';
|
||||
const isConsensusFailed = p.status === 'CONSENSUS_FAILED' || p.status === 'RECOVERY_FAILED';
|
||||
const isRebooting = p.status === 'REBOOTING';
|
||||
const isAborted = p.status === 'ABORTED';
|
||||
|
||||
const agentPills = agents.map(a => {
|
||||
const type = (a.type || 'UNKNOWN').toLowerCase();
|
||||
@@ -5757,10 +5930,37 @@ function renderDashboard(): string {
|
||||
</span>\`;
|
||||
}).join('');
|
||||
|
||||
const consensusAlert = isConsensusFailed ? \`
|
||||
// Rebooting alert (auto-recovery in progress)
|
||||
const rebootingAlert = isRebooting ? \`
|
||||
<div class="consensus-failure-alert" style="border-color: var(--accent-cyan); background: rgba(57, 197, 207, 0.1);" onclick="event.stopPropagation()">
|
||||
<div class="alert-title" style="color: var(--accent-cyan);">Consensus Failure – Pipeline Rebooting</div>
|
||||
<div class="alert-desc">Auto-recovery in progress. A new pipeline is being spawned with failure context.</div>
|
||||
<div class="fallback-options">
|
||||
<div class="fallback-option" onclick="showFailureDetails('\${p.pipeline_id}')">
|
||||
<div>
|
||||
<div class="option-label">Failure Log</div>
|
||||
<div class="option-desc">View what went wrong</div>
|
||||
</div>
|
||||
<button>View Log</button>
|
||||
</div>
|
||||
\${p.recovery_pipeline ? \`
|
||||
<div class="fallback-option" onclick="selectPipeline('\${p.recovery_pipeline}')">
|
||||
<div>
|
||||
<div class="option-label">Recovery Pipeline</div>
|
||||
<div class="option-desc">\${p.recovery_pipeline}</div>
|
||||
</div>
|
||||
<button class="primary">View</button>
|
||||
</div>
|
||||
\` : ''}
|
||||
</div>
|
||||
</div>
|
||||
\` : '';
|
||||
|
||||
// Consensus failed alert (manual action needed - auto-recovery failed or max attempts)
|
||||
const consensusAlert = (isConsensusFailed && !isRebooting) ? \`
|
||||
<div class="consensus-failure-alert" onclick="event.stopPropagation()">
|
||||
<div class="alert-title">Consensus Failed</div>
|
||||
<div class="alert-desc">Agents completed but failed to reach agreement. Choose an action:</div>
|
||||
<div class="alert-title">Consensus Failed – Action Required</div>
|
||||
<div class="alert-desc">Auto-recovery exhausted or failed. Choose a manual action:</div>
|
||||
<div class="fallback-options">
|
||||
<div class="fallback-option" onclick="handleFallback('\${p.pipeline_id}', 'rerun_same')">
|
||||
<div>
|
||||
@@ -5794,15 +5994,34 @@ function renderDashboard(): string {
|
||||
</div>
|
||||
\` : '';
|
||||
|
||||
// Aborted alert
|
||||
const abortedAlert = isAborted ? \`
|
||||
<div class="consensus-failure-alert" style="border-color: var(--accent-red);" onclick="event.stopPropagation()">
|
||||
<div class="alert-title" style="color: var(--accent-red);">Pipeline Aborted</div>
|
||||
<div class="alert-desc">Agents were stuck or exceeded iteration limit. Auto-recovery triggered.</div>
|
||||
<div class="fallback-options">
|
||||
<div class="fallback-option" onclick="showFailureDetails('\${p.pipeline_id}')">
|
||||
<div>
|
||||
<div class="option-label">Failure Log</div>
|
||||
<div class="option-desc">View abort reason</div>
|
||||
</div>
|
||||
<button>View Log</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
\` : '';
|
||||
|
||||
return \`
|
||||
<div class="pipeline-card \${isActive ? 'active' : ''} \${isConsensusFailed ? 'consensus-failed' : ''}" onclick="selectPipeline('\${p.pipeline_id}')">
|
||||
<div class="pipeline-card \${isActive ? 'active' : ''} \${isConsensusFailed ? 'consensus-failed' : ''} \${isRebooting ? 'rebooting' : ''}" onclick="selectPipeline('\${p.pipeline_id}')">
|
||||
<div class="pipeline-header">
|
||||
<span class="pipeline-id">\${p.pipeline_id}</span>
|
||||
<span class="status-badge \${(p.status || 'unknown').toLowerCase().replace(/_/g, '_')}">\${p.status || 'UNKNOWN'}</span>
|
||||
</div>
|
||||
<div class="pipeline-objective">\${p.objective || 'No objective'}</div>
|
||||
<div class="agent-pills">\${agentPills || '<span style="color: var(--text-muted); font-size: 10px;">No agents</span>'}</div>
|
||||
\${rebootingAlert}
|
||||
\${consensusAlert}
|
||||
\${abortedAlert}
|
||||
</div>
|
||||
\`;
|
||||
}).join('');
|
||||
@@ -7642,6 +7861,44 @@ const server = Bun.serve({
|
||||
return new Response(JSON.stringify(result), { headers });
|
||||
}
|
||||
|
||||
// Failure context recording from orchestrator (for auto-recovery)
|
||||
if (path === "/api/pipeline/failure-context" && req.method === "POST") {
|
||||
const body = await req.json() as {
|
||||
pipeline_id: string;
|
||||
task_id: string;
|
||||
failure_reason: string;
|
||||
failure_time: string;
|
||||
iteration_count: number;
|
||||
elapsed_ms: number;
|
||||
metrics: any;
|
||||
gamma_spawned: boolean;
|
||||
error_count: number;
|
||||
recovery_hint: string;
|
||||
};
|
||||
if (!body.pipeline_id) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
|
||||
// Store the failure context in Dragonfly
|
||||
const contextKey = `failure_context:${body.pipeline_id}`;
|
||||
await redis.set(contextKey, JSON.stringify(body));
|
||||
|
||||
// Also log to metrics
|
||||
await redis.hSet(`metrics:${body.pipeline_id}`, {
|
||||
failure_reason: body.failure_reason,
|
||||
failure_time: body.failure_time,
|
||||
iteration_count: String(body.iteration_count),
|
||||
gamma_spawned: body.gamma_spawned ? "true" : "false",
|
||||
error_count: String(body.error_count),
|
||||
recovery_hint: body.recovery_hint
|
||||
});
|
||||
|
||||
await appendPipelineLog(body.pipeline_id, "ORCHESTRATOR",
|
||||
`Failure context recorded: ${body.failure_reason} (${body.iteration_count} iterations, ${body.error_count} errors)`, "WARN");
|
||||
|
||||
return new Response(JSON.stringify({ success: true }), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/observability/handoff" && req.method === "POST") {
|
||||
const body = await req.json() as { pipeline_id: string };
|
||||
if (!body.pipeline_id) {
|
||||
|
||||
Reference in New Issue
Block a user