Add Vault token management and observability integration for multi-agent pipelines
- Vault token issuance per pipeline with 2-hour TTL - Automatic token renewal loop every 30 minutes - Error budget tracking with threshold-based revocation - Observability-driven token revocation for policy violations - Diagnostic pipeline spawning on error threshold breach - Structured handoff reports for error recovery - Agent lifecycle status API - New API endpoints: /api/pipeline/token, /api/pipeline/errors, /api/observability/handoff, /api/observability/diagnostic Orchestrator now reports errors to parent pipeline's observability system via PIPELINE_ID environment variable. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.5
parent
a304895249
commit
8561d13728
@@ -1,6 +1,10 @@
|
||||
/**
|
||||
* Multi-Agent Coordination System - Orchestrator
|
||||
* Manages parallel agent execution, spawn conditions, and metrics
|
||||
*
|
||||
* Environment variables:
|
||||
* - PIPELINE_ID: Parent pipeline ID for error reporting
|
||||
* - TASK_ID: Task ID override
|
||||
*/
|
||||
|
||||
import type { TaskDefinition, CoordinationMetrics, SpawnCondition, AgentRole } from "./types";
|
||||
@@ -21,12 +25,41 @@ function generateId(): string {
|
||||
return "task-" + Math.random().toString(36).slice(2, 8) + "-" + Date.now().toString(36);
|
||||
}
|
||||
|
||||
// Error reporting to parent pipeline's observability system
|
||||
async function reportErrorToObservability(
|
||||
pipelineId: string,
|
||||
errorType: string,
|
||||
severity: "low" | "medium" | "high" | "critical",
|
||||
details: string
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Report to the UI server's error tracking API
|
||||
const response = await fetch("http://localhost:3000/api/pipeline/errors/record", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
pipeline_id: pipelineId,
|
||||
error_type: errorType,
|
||||
severity,
|
||||
details
|
||||
})
|
||||
});
|
||||
if (!response.ok) {
|
||||
console.error(`[ERROR_REPORT] Failed to report error: ${response.status}`);
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Silently fail - don't let error reporting cause more errors
|
||||
console.error(`[ERROR_REPORT] Error reporting failed: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Multi-Agent Orchestrator
|
||||
// =============================================================================
|
||||
|
||||
export class MultiAgentOrchestrator {
|
||||
private taskId: string;
|
||||
private pipelineId?: string;
|
||||
private blackboard!: Blackboard;
|
||||
private stateManager!: AgentStateManager;
|
||||
private spawnController!: SpawnController;
|
||||
@@ -43,12 +76,23 @@ export class MultiAgentOrchestrator {
|
||||
private model: string;
|
||||
private startTime!: number;
|
||||
private monitorInterval?: ReturnType<typeof setInterval>;
|
||||
private errorCount: number = 0;
|
||||
|
||||
constructor(model: string = "anthropic/claude-sonnet-4") {
|
||||
this.taskId = generateId();
|
||||
// Use environment variable for task ID if provided
|
||||
this.taskId = process.env.TASK_ID || generateId();
|
||||
this.pipelineId = process.env.PIPELINE_ID;
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
private async reportError(errorType: string, severity: "low" | "medium" | "high" | "critical", details: string): Promise<void> {
|
||||
this.errorCount++;
|
||||
if (this.pipelineId) {
|
||||
await reportErrorToObservability(this.pipelineId, errorType, severity, details);
|
||||
}
|
||||
this.log(`ERROR [${severity}] ${errorType}: ${details}`);
|
||||
}
|
||||
|
||||
private log(msg: string) {
|
||||
const elapsed = this.startTime ? ((Date.now() - this.startTime) / 1000).toFixed(1) : "0.0";
|
||||
console.log(`[${elapsed}s] [ORCHESTRATOR] ${msg}`);
|
||||
@@ -60,6 +104,9 @@ export class MultiAgentOrchestrator {
|
||||
console.log("\n" + "=".repeat(70));
|
||||
console.log("MULTI-AGENT COORDINATION SYSTEM");
|
||||
console.log("Task ID: " + this.taskId);
|
||||
if (this.pipelineId) {
|
||||
console.log("Pipeline ID: " + this.pipelineId);
|
||||
}
|
||||
console.log("Model: " + this.model);
|
||||
console.log("=".repeat(70) + "\n");
|
||||
|
||||
@@ -186,12 +233,12 @@ export class MultiAgentOrchestrator {
|
||||
// Run agents in parallel
|
||||
this.log("Launching ALPHA and BETA in parallel...");
|
||||
|
||||
const alphaPromise = this.alphaAgent.run(task).catch(e => {
|
||||
this.log(`ALPHA error: ${e.message}`);
|
||||
const alphaPromise = this.alphaAgent.run(task).catch(async e => {
|
||||
await this.reportError("agent_failure", "high", `ALPHA error: ${e.message}`);
|
||||
});
|
||||
|
||||
const betaPromise = this.betaAgent.run(task).catch(e => {
|
||||
this.log(`BETA error: ${e.message}`);
|
||||
const betaPromise = this.betaAgent.run(task).catch(async e => {
|
||||
await this.reportError("agent_failure", "high", `BETA error: ${e.message}`);
|
||||
});
|
||||
|
||||
// Wait for initial agents to complete (or timeout)
|
||||
@@ -220,8 +267,8 @@ export class MultiAgentOrchestrator {
|
||||
// If GAMMA was spawned, run it
|
||||
if (this.gammaAgent) {
|
||||
this.log("Running GAMMA for resolution...");
|
||||
await this.gammaAgent.run(task).catch(e => {
|
||||
this.log(`GAMMA error: ${e.message}`);
|
||||
await this.gammaAgent.run(task).catch(async e => {
|
||||
await this.reportError("agent_failure", "high", `GAMMA error: ${e.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -404,6 +451,12 @@ The solution should consider fault tolerance, data consistency, and cost optimiz
|
||||
} catch (e: any) {
|
||||
console.error("Orchestrator error:", e.message);
|
||||
exitCode = 1;
|
||||
|
||||
// Report critical error to observability if pipeline ID is set
|
||||
const pipelineId = process.env.PIPELINE_ID;
|
||||
if (pipelineId) {
|
||||
await reportErrorToObservability(pipelineId, "orchestrator_failure", "critical", e.message);
|
||||
}
|
||||
} finally {
|
||||
await orchestrator.cleanup();
|
||||
// Explicitly exit to ensure all connections are closed
|
||||
|
||||
Reference in New Issue
Block a user