Add Vault token management and observability integration for multi-agent pipelines

- Vault token issuance per pipeline with 2-hour TTL
- Automatic token renewal loop every 30 minutes
- Error budget tracking with threshold-based revocation
- Observability-driven token revocation for policy violations
- Diagnostic pipeline spawning on error threshold breach
- Structured handoff reports for error recovery
- Agent lifecycle status API
- New API endpoints: /api/pipeline/token, /api/pipeline/errors,
  /api/observability/handoff, /api/observability/diagnostic

Orchestrator now reports errors to parent pipeline's observability
system via PIPELINE_ID environment variable.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
2026-01-24 17:45:20 -05:00
co-authored by Claude Opus 4.5
parent a304895249
commit 8561d13728
5 changed files with 1252 additions and 22 deletions
+60 -7
View File
@@ -1,6 +1,10 @@
/**
* Multi-Agent Coordination System - Orchestrator
* Manages parallel agent execution, spawn conditions, and metrics
*
* Environment variables:
* - PIPELINE_ID: Parent pipeline ID for error reporting
* - TASK_ID: Task ID override
*/
import type { TaskDefinition, CoordinationMetrics, SpawnCondition, AgentRole } from "./types";
@@ -21,12 +25,41 @@ function generateId(): string {
return "task-" + Math.random().toString(36).slice(2, 8) + "-" + Date.now().toString(36);
}
// Error reporting to parent pipeline's observability system
async function reportErrorToObservability(
pipelineId: string,
errorType: string,
severity: "low" | "medium" | "high" | "critical",
details: string
): Promise<void> {
try {
// Report to the UI server's error tracking API
const response = await fetch("http://localhost:3000/api/pipeline/errors/record", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
pipeline_id: pipelineId,
error_type: errorType,
severity,
details
})
});
if (!response.ok) {
console.error(`[ERROR_REPORT] Failed to report error: ${response.status}`);
}
} catch (e: any) {
// Silently fail - don't let error reporting cause more errors
console.error(`[ERROR_REPORT] Error reporting failed: ${e.message}`);
}
}
// =============================================================================
// Multi-Agent Orchestrator
// =============================================================================
export class MultiAgentOrchestrator {
private taskId: string;
private pipelineId?: string;
private blackboard!: Blackboard;
private stateManager!: AgentStateManager;
private spawnController!: SpawnController;
@@ -43,12 +76,23 @@ export class MultiAgentOrchestrator {
private model: string;
private startTime!: number;
private monitorInterval?: ReturnType<typeof setInterval>;
private errorCount: number = 0;
constructor(model: string = "anthropic/claude-sonnet-4") {
this.taskId = generateId();
// Use environment variable for task ID if provided
this.taskId = process.env.TASK_ID || generateId();
this.pipelineId = process.env.PIPELINE_ID;
this.model = model;
}
private async reportError(errorType: string, severity: "low" | "medium" | "high" | "critical", details: string): Promise<void> {
this.errorCount++;
if (this.pipelineId) {
await reportErrorToObservability(this.pipelineId, errorType, severity, details);
}
this.log(`ERROR [${severity}] ${errorType}: ${details}`);
}
private log(msg: string) {
const elapsed = this.startTime ? ((Date.now() - this.startTime) / 1000).toFixed(1) : "0.0";
console.log(`[${elapsed}s] [ORCHESTRATOR] ${msg}`);
@@ -60,6 +104,9 @@ export class MultiAgentOrchestrator {
console.log("\n" + "=".repeat(70));
console.log("MULTI-AGENT COORDINATION SYSTEM");
console.log("Task ID: " + this.taskId);
if (this.pipelineId) {
console.log("Pipeline ID: " + this.pipelineId);
}
console.log("Model: " + this.model);
console.log("=".repeat(70) + "\n");
@@ -186,12 +233,12 @@ export class MultiAgentOrchestrator {
// Run agents in parallel
this.log("Launching ALPHA and BETA in parallel...");
const alphaPromise = this.alphaAgent.run(task).catch(e => {
this.log(`ALPHA error: ${e.message}`);
const alphaPromise = this.alphaAgent.run(task).catch(async e => {
await this.reportError("agent_failure", "high", `ALPHA error: ${e.message}`);
});
const betaPromise = this.betaAgent.run(task).catch(e => {
this.log(`BETA error: ${e.message}`);
const betaPromise = this.betaAgent.run(task).catch(async e => {
await this.reportError("agent_failure", "high", `BETA error: ${e.message}`);
});
// Wait for initial agents to complete (or timeout)
@@ -220,8 +267,8 @@ export class MultiAgentOrchestrator {
// If GAMMA was spawned, run it
if (this.gammaAgent) {
this.log("Running GAMMA for resolution...");
await this.gammaAgent.run(task).catch(e => {
this.log(`GAMMA error: ${e.message}`);
await this.gammaAgent.run(task).catch(async e => {
await this.reportError("agent_failure", "high", `GAMMA error: ${e.message}`);
});
}
@@ -404,6 +451,12 @@ The solution should consider fault tolerance, data consistency, and cost optimiz
} catch (e: any) {
console.error("Orchestrator error:", e.message);
exitCode = 1;
// Report critical error to observability if pipeline ID is set
const pipelineId = process.env.PIPELINE_ID;
if (pipelineId) {
await reportErrorToObservability(pipelineId, "orchestrator_failure", "critical", e.message);
}
} finally {
await orchestrator.cleanup();
// Explicitly exit to ensure all connections are closed