Implement real auto-recovery with handoff chain
Orchestrator changes:
- Add dumpAgentHandoff() to dump proposals/analysis before abort
- Add loadRecoveryContext() to load inherited context on recovery runs
- Add preseedBlackboard() to pre-seed inherited proposals
- Force-spawn GAMMA immediately on recovery runs
- Track isRecoveryRun, recoveryAttempt, inheritedContext, forceGamma
Server changes:
- Update recordConsensusFailure() to read orchestrator handoff JSON
- Add collectFromBlackboard() helper as fallback
- Update triggerAutoRecovery() with comprehensive context passing
- Store inherited_handoff reference for recovery pipelines
- Track retry_count, abort_reason, handoff_ref in recovery:* keys
- Add recovery badge and prior pipeline link in UI
Test coverage:
- test_auto_recovery.py: 6 unit tests
- test_e2e_auto_recovery.py: 5 E2E tests (handoff dump, recovery
pipeline creation, inherited context, retry tracking, status update)
Redis tracking keys:
- handoff:{pipeline_id}:agents - orchestrator dumps proposals here
- handoff:{recovery_id}:inherited - recovery pipeline inherits from
- recovery:{pipeline_id} - retry_count, abort_reason, handoff_ref
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
a19535b580
commit
c96919fe35
+180
-43
@@ -1516,34 +1516,27 @@ const FALLBACK_OPTIONS: FallbackOption[] = [
|
||||
}
|
||||
];
|
||||
|
||||
async function recordConsensusFailure(
|
||||
pipelineId: string,
|
||||
taskId: string,
|
||||
metrics: any
|
||||
): Promise<ConsensusFailureContext> {
|
||||
const pipelineKey = `pipeline:${pipelineId}`;
|
||||
const pipelineData = await redis.hGetAll(pipelineKey);
|
||||
|
||||
// Get run number (increment if retrying)
|
||||
const prevRunNumber = parseInt(pipelineData.run_number || "0");
|
||||
const runNumber = prevRunNumber + 1;
|
||||
|
||||
// Collect all context for the failed run
|
||||
const context: ConsensusFailureContext = {
|
||||
pipeline_id: pipelineId,
|
||||
task_id: taskId,
|
||||
objective: pipelineData.objective || "",
|
||||
failure_time: new Date().toISOString(),
|
||||
metrics: metrics,
|
||||
proposals: [],
|
||||
agent_states: [],
|
||||
conflict_history: [],
|
||||
blackboard_snapshot: {},
|
||||
run_number: runNumber
|
||||
};
|
||||
|
||||
// Collect proposals from blackboard (if available in Redis)
|
||||
// Helper: Collect context from blackboard (fallback when handoff not available)
|
||||
async function collectFromBlackboard(context: ConsensusFailureContext, taskId: string): Promise<void> {
|
||||
try {
|
||||
// Collect proposals from blackboard solutions section
|
||||
const solutionsData = await redis.hGetAll(`blackboard:${taskId}:solutions`);
|
||||
for (const [key, value] of Object.entries(solutionsData)) {
|
||||
try {
|
||||
const entry = JSON.parse(value as string);
|
||||
context.proposals.push({
|
||||
agent: entry.author,
|
||||
key: entry.key,
|
||||
value: entry.value,
|
||||
version: entry.version,
|
||||
timestamp: entry.timestamp
|
||||
});
|
||||
} catch {
|
||||
context.proposals.push({ key, raw: value });
|
||||
}
|
||||
}
|
||||
|
||||
// Also check old key format
|
||||
const proposalKeys = await redis.keys(`blackboard:${taskId}:solutions:*`);
|
||||
for (const key of proposalKeys) {
|
||||
const proposal = await redis.get(key);
|
||||
@@ -1585,7 +1578,73 @@ async function recordConsensusFailure(
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error(`[CONSENSUS] Error collecting context: ${e.message}`);
|
||||
console.error(`[CONSENSUS] Error collecting from blackboard: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function recordConsensusFailure(
|
||||
pipelineId: string,
|
||||
taskId: string,
|
||||
metrics: any
|
||||
): Promise<ConsensusFailureContext> {
|
||||
const pipelineKey = `pipeline:${pipelineId}`;
|
||||
const pipelineData = await redis.hGetAll(pipelineKey);
|
||||
|
||||
// Get run number (increment if retrying)
|
||||
const prevRunNumber = parseInt(pipelineData.run_number || "0");
|
||||
const runNumber = prevRunNumber + 1;
|
||||
|
||||
// Collect all context for the failed run
|
||||
const context: ConsensusFailureContext = {
|
||||
pipeline_id: pipelineId,
|
||||
task_id: taskId,
|
||||
objective: pipelineData.objective || "",
|
||||
failure_time: new Date().toISOString(),
|
||||
metrics: metrics,
|
||||
proposals: [],
|
||||
agent_states: [],
|
||||
conflict_history: [],
|
||||
blackboard_snapshot: {},
|
||||
run_number: runNumber
|
||||
};
|
||||
|
||||
// FIRST: Try to read the structured handoff JSON from the orchestrator
|
||||
try {
|
||||
const handoffKey = `handoff:${pipelineId}:agents`;
|
||||
const handoffData = await redis.get(handoffKey);
|
||||
|
||||
if (handoffData) {
|
||||
const handoff = JSON.parse(handoffData);
|
||||
console.log(`[CONSENSUS] Found orchestrator handoff: ${handoff.proposals?.length || 0} proposals`);
|
||||
|
||||
// Use handoff data directly - it's more complete
|
||||
context.proposals = handoff.proposals || [];
|
||||
context.agent_states = handoff.agent_states || [];
|
||||
context.conflict_history = handoff.message_summary?.alpha_last_messages?.concat(
|
||||
handoff.message_summary?.beta_last_messages || [],
|
||||
handoff.message_summary?.gamma_last_messages || []
|
||||
).filter((m: any) => m.type === "CONFLICT" || m.type === "PROPOSAL" || m.type === "VOTE") || [];
|
||||
|
||||
// Store synthesis attempts and problem analysis in blackboard snapshot
|
||||
context.blackboard_snapshot = {
|
||||
synthesis: handoff.synthesis_attempts || [],
|
||||
consensus: handoff.consensus_state || [],
|
||||
problem: handoff.problem_analysis || [],
|
||||
recovery_hints: handoff.recovery_hints || []
|
||||
};
|
||||
|
||||
// Store handoff reference
|
||||
(context as any).handoff_ref = handoffKey;
|
||||
(context as any).iteration_count = handoff.iteration_count;
|
||||
(context as any).gamma_active = handoff.gamma_active;
|
||||
} else {
|
||||
console.log(`[CONSENSUS] No handoff found, falling back to blackboard scan`);
|
||||
// Fallback to blackboard scan (old method)
|
||||
await collectFromBlackboard(context, taskId);
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error(`[CONSENSUS] Error reading handoff, falling back: ${e.message}`);
|
||||
await collectFromBlackboard(context, taskId);
|
||||
}
|
||||
|
||||
// Store the failure context in Dragonfly
|
||||
@@ -1802,20 +1861,47 @@ async function triggerAutoRecovery(
|
||||
// Create a new recovery pipeline
|
||||
const newPipelineId = `pipeline-recovery-${Date.now().toString(36)}`;
|
||||
|
||||
// Prepare context summary for new agents
|
||||
// Get handoff reference from original pipeline (if available)
|
||||
const handoffRef = (failureContext as any).handoff_ref || `handoff:${originalPipelineId}:agents`;
|
||||
const iterationCount = (failureContext as any).iteration_count || 0;
|
||||
const gammaWasActive = (failureContext as any).gamma_active || false;
|
||||
|
||||
// Prepare comprehensive context summary for new agents
|
||||
const contextSummary = {
|
||||
prior_run: runNumber,
|
||||
prior_pipeline: originalPipelineId,
|
||||
handoff_ref: handoffRef,
|
||||
failure_reason: failureContext.metrics?.abort_reason || "consensus_failed",
|
||||
prior_proposals: failureContext.proposals?.slice(0, 3) || [], // Top 3 proposals
|
||||
prior_conflicts: failureContext.conflict_history?.slice(-5) || [], // Last 5 conflicts
|
||||
iteration_count: iterationCount,
|
||||
gamma_was_active: gammaWasActive,
|
||||
|
||||
// Include actual proposals (not just references)
|
||||
prior_proposals: failureContext.proposals?.slice(0, 5) || [], // Top 5 proposals
|
||||
prior_synthesis: failureContext.blackboard_snapshot?.synthesis || [],
|
||||
prior_conflicts: failureContext.conflict_history?.slice(-10) || [], // Last 10 conflicts
|
||||
|
||||
recovery_hints: [
|
||||
"Previous agents failed to reach consensus",
|
||||
`Previous run aborted after ${iterationCount} iterations`,
|
||||
gammaWasActive ? "GAMMA was active but could not resolve conflicts" : "GAMMA was not spawned - will be forced this time",
|
||||
`${failureContext.proposals?.length || 0} proposals were generated`,
|
||||
"Review prior proposals for common ground",
|
||||
"Consider a different approach if prior attempts converged on same solution"
|
||||
"Consider synthesizing the best elements of prior proposals"
|
||||
]
|
||||
};
|
||||
|
||||
// Store full handoff in Redis for the new pipeline
|
||||
const newHandoffKey = `handoff:${newPipelineId}:inherited`;
|
||||
await redis.set(newHandoffKey, JSON.stringify({
|
||||
from_pipeline: originalPipelineId,
|
||||
from_handoff: handoffRef,
|
||||
inherited_at: new Date().toISOString(),
|
||||
proposals: failureContext.proposals,
|
||||
synthesis_attempts: failureContext.blackboard_snapshot?.synthesis,
|
||||
consensus_state: failureContext.blackboard_snapshot?.consensus,
|
||||
agent_states: failureContext.agent_states,
|
||||
recovery_hints: contextSummary.recovery_hints
|
||||
}), { EX: 86400 }); // 24hr TTL
|
||||
|
||||
await redis.hSet(`pipeline:${newPipelineId}`, {
|
||||
task_id: taskId,
|
||||
objective: objective,
|
||||
@@ -1825,34 +1911,57 @@ async function triggerAutoRecovery(
|
||||
parent_pipeline: originalPipelineId,
|
||||
is_recovery: "true",
|
||||
recovery_attempt: String(runNumber + 1),
|
||||
run_number: String(runNumber + 1),
|
||||
prior_context: JSON.stringify(contextSummary),
|
||||
inherited_handoff: newHandoffKey,
|
||||
force_gamma: "true", // Always spawn GAMMA on recovery attempts
|
||||
model: model,
|
||||
timeout: String(timeout),
|
||||
auto_continue: "true"
|
||||
});
|
||||
|
||||
// Update original pipeline
|
||||
// Update original pipeline with detailed recovery tracking
|
||||
await redis.hSet(`pipeline:${originalPipelineId}`, {
|
||||
status: "REBOOTING",
|
||||
recovery_pipeline: newPipelineId
|
||||
recovery_pipeline: newPipelineId,
|
||||
recovery_triggered_at: new Date().toISOString()
|
||||
});
|
||||
|
||||
// Log the handoff reason to Dragonfly metrics
|
||||
// Track retry metrics in Dragonfly
|
||||
await redis.hSet(`recovery:${originalPipelineId}`, {
|
||||
retry_count: String(runNumber + 1),
|
||||
abort_reason: failureContext.metrics?.abort_reason || "consensus_failed",
|
||||
latest_recovery: newPipelineId,
|
||||
handoff_ref: handoffRef,
|
||||
proposals_passed: String(failureContext.proposals?.length || 0),
|
||||
last_attempt: new Date().toISOString()
|
||||
});
|
||||
|
||||
// Log detailed handoff reason to Dragonfly metrics
|
||||
await redis.hSet(`handoff:${originalPipelineId}`, {
|
||||
to_pipeline: newPipelineId,
|
||||
reason: failureContext.metrics?.abort_reason || "consensus_failed",
|
||||
handoff_time: new Date().toISOString(),
|
||||
context_size: JSON.stringify(contextSummary).length,
|
||||
proposals_passed: failureContext.proposals?.length || 0
|
||||
context_size: String(JSON.stringify(contextSummary).length),
|
||||
proposals_passed: String(failureContext.proposals?.length || 0),
|
||||
iteration_count: String(iterationCount),
|
||||
gamma_was_active: gammaWasActive ? "true" : "false"
|
||||
});
|
||||
|
||||
await appendPipelineLog(newPipelineId, "SYSTEM",
|
||||
`Recovery pipeline started (attempt ${runNumber + 1}/${MAX_AUTO_RECOVERY}). GAMMA mediator will be spawned.`, "INFO");
|
||||
`Recovery pipeline started (attempt ${runNumber + 1}/${MAX_AUTO_RECOVERY}). GAMMA mediator will be force-spawned.`, "INFO");
|
||||
|
||||
await appendPipelineLog(newPipelineId, "CONTEXT",
|
||||
`Prior run had ${failureContext.proposals?.length || 0} proposals, ` +
|
||||
`${failureContext.conflict_history?.length || 0} conflicts. Force-spawning GAMMA.`, "INFO");
|
||||
`Inherited from ${originalPipelineId}: ${failureContext.proposals?.length || 0} proposals, ` +
|
||||
`${failureContext.conflict_history?.length || 0} conflicts, ${iterationCount} iterations.`, "INFO");
|
||||
|
||||
if (failureContext.proposals && failureContext.proposals.length > 0) {
|
||||
// Log summary of inherited proposals
|
||||
const proposalSummary = failureContext.proposals.slice(0, 3).map((p: any, i: number) =>
|
||||
`${i + 1}. [${p.agent}] ${typeof p.value === 'string' ? p.value.slice(0, 100) : JSON.stringify(p.value).slice(0, 100)}...`
|
||||
).join("\n");
|
||||
await appendPipelineLog(newPipelineId, "CONTEXT", `Top inherited proposals:\n${proposalSummary}`, "INFO");
|
||||
}
|
||||
|
||||
// Trigger orchestration with GAMMA hint in objective
|
||||
triggerOrchestration(
|
||||
@@ -2067,6 +2176,12 @@ async function getActivePipelines(): Promise<any[]> {
|
||||
failure_reason: data.failure_reason || null,
|
||||
run_number: data.run_number ? parseInt(data.run_number) : 1,
|
||||
prior_pipeline: data.prior_pipeline || null,
|
||||
// Recovery tracking
|
||||
is_recovery: data.is_recovery || null,
|
||||
parent_pipeline: data.parent_pipeline || null,
|
||||
recovery_attempt: data.recovery_attempt ? parseInt(data.recovery_attempt) : null,
|
||||
force_gamma: data.force_gamma || null,
|
||||
inherited_handoff: data.inherited_handoff || null,
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
@@ -4069,6 +4184,8 @@ function renderDashboard(): string {
|
||||
.status-badge.rebooting { background: rgba(57, 197, 207, 0.2); color: var(--accent-cyan); animation: pulse 1.5s infinite; }
|
||||
.status-badge.aborted { background: rgba(248, 81, 73, 0.3); color: var(--accent-red); border: 1px solid var(--accent-red); }
|
||||
.status-badge.recovery_failed { background: rgba(248, 81, 73, 0.4); color: #ff6b6b; border: 1px solid #ff6b6b; }
|
||||
.status-badge.recovery { background: rgba(139, 92, 246, 0.2); color: var(--accent-purple); border: 1px solid var(--accent-purple); }
|
||||
.pipeline-card.recovery-run { border-left: 3px solid var(--accent-purple); }
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
@@ -5920,6 +6037,8 @@ function renderDashboard(): string {
|
||||
const isConsensusFailed = p.status === 'CONSENSUS_FAILED' || p.status === 'RECOVERY_FAILED';
|
||||
const isRebooting = p.status === 'REBOOTING';
|
||||
const isAborted = p.status === 'ABORTED';
|
||||
const isRecoveryRun = p.is_recovery === 'true' || p.recovery_attempt;
|
||||
const runNumber = p.run_number || p.recovery_attempt || 1;
|
||||
|
||||
const agentPills = agents.map(a => {
|
||||
const type = (a.type || 'UNKNOWN').toLowerCase();
|
||||
@@ -6011,13 +6130,31 @@ function renderDashboard(): string {
|
||||
</div>
|
||||
\` : '';
|
||||
|
||||
// Recovery run indicator
|
||||
const recoveryBadge = isRecoveryRun ? \`
|
||||
<span class="status-badge recovery" title="Recovery run \${runNumber} - inheriting context from prior run">
|
||||
RETRY \${runNumber}
|
||||
</span>
|
||||
\` : '';
|
||||
|
||||
// Prior pipeline link for recovery runs
|
||||
const priorPipelineLink = (isRecoveryRun && p.parent_pipeline) ? \`
|
||||
<div style="font-size: 10px; color: var(--text-muted); margin-top: 4px;">
|
||||
Prior: <span style="color: var(--accent-cyan); cursor: pointer;" onclick="event.stopPropagation(); selectPipeline('\${p.parent_pipeline}')">\${p.parent_pipeline}</span>
|
||||
</div>
|
||||
\` : '';
|
||||
|
||||
return \`
|
||||
<div class="pipeline-card \${isActive ? 'active' : ''} \${isConsensusFailed ? 'consensus-failed' : ''} \${isRebooting ? 'rebooting' : ''}" onclick="selectPipeline('\${p.pipeline_id}')">
|
||||
<div class="pipeline-card \${isActive ? 'active' : ''} \${isConsensusFailed ? 'consensus-failed' : ''} \${isRebooting ? 'rebooting' : ''} \${isRecoveryRun ? 'recovery-run' : ''}" onclick="selectPipeline('\${p.pipeline_id}')">
|
||||
<div class="pipeline-header">
|
||||
<span class="pipeline-id">\${p.pipeline_id}</span>
|
||||
<span class="status-badge \${(p.status || 'unknown').toLowerCase().replace(/_/g, '_')}">\${p.status || 'UNKNOWN'}</span>
|
||||
<div style="display: flex; gap: 6px; align-items: center;">
|
||||
\${recoveryBadge}
|
||||
<span class="status-badge \${(p.status || 'unknown').toLowerCase().replace(/_/g, '_')}">\${p.status || 'UNKNOWN'}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pipeline-objective">\${p.objective || 'No objective'}</div>
|
||||
\${priorPipelineLink}
|
||||
<div class="agent-pills">\${agentPills || '<span style="color: var(--text-muted); font-size: 10px;">No agents</span>'}</div>
|
||||
\${rebootingAlert}
|
||||
\${consensusAlert}
|
||||
|
||||
Reference in New Issue
Block a user