Implement real supervisor-driven auto-recovery

Orchestrator changes:
- Force-spawn GAMMA on iteration_limit before abort
- GAMMA.synthesize() creates emergency handoff payload
- loadRecoveryContext() logs "Resuming from {task_id} handoff"
- POST to /api/pipeline/log for resume message visibility

AgentGamma changes:
- Add synthesize() method for emergency abort synthesis
- Merges existing proposals into coherent handoff
- Stores as synthesis_type: "abort_recovery"

Server changes:
- Add POST /api/pipeline/log endpoint for orchestrator logging
- Recovery pipeline properly inherits GAMMA synthesis

Test coverage:
- test_auto_recovery.py: 6 unit tests
- test_e2e_auto_recovery.py: 5 E2E tests
- test_supervisor_recovery.py: 3 supervisor tests
  - Success on attempt 2 (recovery works)
  - Max failures (3 retries then FAILED)
  - Success on attempt 1 (no recovery needed)

Recovery flow:
1. iteration_limit triggers
2. GAMMA force-spawned for emergency synthesis
3. Handoff dumped with GAMMA synthesis
4. Exit code 3 triggers auto-recovery
5. Recovery pipeline loads handoff
6. Logs "Resuming from {prior_pipeline} handoff"
7. Repeat up to 3 times or until success

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
2026-01-24 19:47:56 -05:00
co-authored by Claude Opus 4.5
parent c96919fe35
commit ef18567674
4 changed files with 617 additions and 0 deletions
+58
View File
@@ -219,6 +219,10 @@ export class MultiAgentOrchestrator {
this.recoveryAttempt = parseInt(pipelineData.recovery_attempt || "1");
this.forceGamma = pipelineData.force_gamma === "true";
// Get the prior task ID for the resume message
let priorTaskId = "unknown";
let priorPipelineId = pipelineData.parent_pipeline || "unknown";
console.log("\n" + "=".repeat(70));
console.log(`RECOVERY RUN - Attempt ${this.recoveryAttempt}`);
console.log("=".repeat(70));
@@ -229,9 +233,25 @@ export class MultiAgentOrchestrator {
const inheritedData = await redis.get(inheritedKey);
if (inheritedData) {
this.inheritedContext = JSON.parse(inheritedData);
priorTaskId = this.inheritedContext.from_pipeline || priorPipelineId;
// LOG THE RESUME MESSAGE
console.log(`\n>>> Resuming from ${priorTaskId} handoff <<<\n`);
console.log(`Loaded inherited context from: ${inheritedKey}`);
console.log(`- Prior proposals: ${this.inheritedContext.proposals?.length || 0}`);
console.log(`- Prior synthesis attempts: ${this.inheritedContext.synthesis_attempts?.length || 0}`);
// Check for GAMMA emergency synthesis
if (this.inheritedContext.synthesis_attempts?.length > 0) {
const emergencySynth = this.inheritedContext.synthesis_attempts.find(
(s: any) => s.synthesis_type === "abort_recovery"
);
if (emergencySynth) {
console.log(`- GAMMA emergency synthesis found: ${emergencySynth.key_insights?.length || 0} insights`);
}
}
console.log(`- Recovery hints:`);
this.inheritedContext.recovery_hints?.forEach((hint: string, i: number) => {
console.log(` ${i + 1}. ${hint}`);
@@ -252,6 +272,20 @@ export class MultiAgentOrchestrator {
}
console.log("=".repeat(70) + "\n");
// Log to pipeline log as well
try {
await fetch("http://localhost:3000/api/pipeline/log", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
pipeline_id: this.pipelineId,
agent: "ORCHESTRATOR",
message: `Resuming from ${priorTaskId} handoff (attempt ${this.recoveryAttempt}, ${this.inheritedContext?.proposals?.length || 0} inherited proposals)`,
level: "INFO"
})
});
} catch { /* ignore logging errors */ }
}
} catch (e: any) {
this.log(`Warning: Could not load recovery context: ${e.message}`);
@@ -646,6 +680,30 @@ export class MultiAgentOrchestrator {
clearInterval(this.monitorInterval);
}
// CRITICAL: Force-spawn GAMMA to synthesize existing proposals before abort
if (!this.gammaAgent) {
this.log("ABORT RECOVERY: Force-spawning GAMMA to synthesize proposals before handoff...");
const forceReason: SpawnCondition = {
type: "STUCK",
threshold: 0,
current_value: 1,
triggered: true,
description: `Force-spawned on ${abortReason} for synthesis before abort`
};
await this.spawnGamma(forceReason);
// Give GAMMA a short window to synthesize (10 seconds max)
if (this.gammaAgent) {
this.log("GAMMA synthesizing proposals for handoff...");
const synthTimeout = new Promise<void>(resolve => setTimeout(resolve, 10000));
await Promise.race([
this.gammaAgent.synthesize(task).catch(e => this.log(`GAMMA synthesis error: ${e.message}`)),
synthTimeout
]);
this.log("GAMMA synthesis complete (or timed out)");
}
}
// CRITICAL: Dump all agent proposals/analysis to handoff JSON BEFORE revoking tokens
this.log("Dumping agent handoff data for recovery pipeline...");
await this.dumpAgentHandoff();