Add Vault token management and observability integration for multi-agent pipelines
- Vault token issuance per pipeline with 2-hour TTL - Automatic token renewal loop every 30 minutes - Error budget tracking with threshold-based revocation - Observability-driven token revocation for policy violations - Diagnostic pipeline spawning on error threshold breach - Structured handoff reports for error recovery - Agent lifecycle status API - New API endpoints: /api/pipeline/token, /api/pipeline/errors, /api/observability/handoff, /api/observability/diagnostic Orchestrator now reports errors to parent pipeline's observability system via PIPELINE_ID environment variable. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
profit
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
a304895249
commit
8561d13728
+635
-1
@@ -983,6 +983,502 @@ async function getBlackboardSolutions(taskId: string): Promise<any[]> {
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Vault Token Management for Pipelines
|
||||
// =============================================================================
|
||||
|
||||
interface VaultTokenInfo {
|
||||
token: string;
|
||||
accessor: string;
|
||||
ttl: number;
|
||||
created_at: string;
|
||||
renewable: boolean;
|
||||
policies: string[];
|
||||
}
|
||||
|
||||
interface PipelineTokenStatus {
|
||||
pipeline_id: string;
|
||||
token_active: boolean;
|
||||
issued_at?: string;
|
||||
expires_at?: string;
|
||||
last_renewed?: string;
|
||||
revoked?: boolean;
|
||||
revoke_reason?: string;
|
||||
}
|
||||
|
||||
// Error budget tracking
|
||||
interface ErrorBudget {
|
||||
pipeline_id: string;
|
||||
total_errors: number;
|
||||
errors_per_minute: number;
|
||||
last_error_at?: string;
|
||||
threshold_exceeded: boolean;
|
||||
error_types: Record<string, number>;
|
||||
}
|
||||
|
||||
const ERROR_THRESHOLDS = {
|
||||
max_errors_per_minute: 5,
|
||||
max_total_errors: 20,
|
||||
stuck_timeout_seconds: 60,
|
||||
critical_violation_immediate: true,
|
||||
};
|
||||
|
||||
// Track error budgets in memory (also persisted to Redis)
|
||||
const errorBudgets: Map<string, ErrorBudget> = new Map();
|
||||
|
||||
async function issuePipelineToken(pipelineId: string): Promise<VaultTokenInfo | null> {
|
||||
try {
|
||||
const initKeys = await Bun.file("/opt/vault/init-keys.json").json();
|
||||
const rootToken = initKeys.root_token;
|
||||
|
||||
// Create a pipeline-specific token with limited TTL and policies
|
||||
const tokenRequest = {
|
||||
policies: ["pipeline-agent"],
|
||||
ttl: "2h",
|
||||
renewable: true,
|
||||
display_name: `pipeline-${pipelineId}`,
|
||||
meta: {
|
||||
pipeline_id: pipelineId,
|
||||
created_by: "orchestrator"
|
||||
}
|
||||
};
|
||||
|
||||
const proc = Bun.spawn(["curl", "-sk", "-X", "POST",
|
||||
"-H", `X-Vault-Token: ${rootToken}`,
|
||||
"-d", JSON.stringify(tokenRequest),
|
||||
"https://127.0.0.1:8200/v1/auth/token/create"
|
||||
], { stdout: "pipe" });
|
||||
|
||||
const text = await new Response(proc.stdout).text();
|
||||
const result = JSON.parse(text);
|
||||
|
||||
if (result.auth) {
|
||||
const tokenInfo: VaultTokenInfo = {
|
||||
token: result.auth.client_token,
|
||||
accessor: result.auth.accessor,
|
||||
ttl: result.auth.lease_duration,
|
||||
created_at: new Date().toISOString(),
|
||||
renewable: result.auth.renewable,
|
||||
policies: result.auth.policies
|
||||
};
|
||||
|
||||
// Store token info in Redis (encrypted reference, not actual token)
|
||||
await redis.hSet(`pipeline:${pipelineId}:vault`, {
|
||||
accessor: tokenInfo.accessor,
|
||||
issued_at: tokenInfo.created_at,
|
||||
expires_at: new Date(Date.now() + tokenInfo.ttl * 1000).toISOString(),
|
||||
renewable: tokenInfo.renewable ? "true" : "false",
|
||||
policies: JSON.stringify(tokenInfo.policies),
|
||||
status: "active"
|
||||
});
|
||||
|
||||
broadcastUpdate("token_issued", {
|
||||
pipeline_id: pipelineId,
|
||||
accessor: tokenInfo.accessor,
|
||||
expires_at: new Date(Date.now() + tokenInfo.ttl * 1000).toISOString()
|
||||
});
|
||||
|
||||
return tokenInfo;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (e: any) {
|
||||
console.error(`[VAULT] Error issuing token for pipeline ${pipelineId}:`, e.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function renewPipelineToken(pipelineId: string): Promise<boolean> {
|
||||
try {
|
||||
const tokenData = await redis.hGetAll(`pipeline:${pipelineId}:vault`);
|
||||
if (!tokenData.accessor || tokenData.status !== "active") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const initKeys = await Bun.file("/opt/vault/init-keys.json").json();
|
||||
const rootToken = initKeys.root_token;
|
||||
|
||||
// Renew by accessor
|
||||
const proc = Bun.spawn(["curl", "-sk", "-X", "POST",
|
||||
"-H", `X-Vault-Token: ${rootToken}`,
|
||||
"-d", JSON.stringify({ accessor: tokenData.accessor }),
|
||||
"https://127.0.0.1:8200/v1/auth/token/renew-accessor"
|
||||
], { stdout: "pipe" });
|
||||
|
||||
const text = await new Response(proc.stdout).text();
|
||||
const result = JSON.parse(text);
|
||||
|
||||
if (result.auth) {
|
||||
const newExpiry = new Date(Date.now() + result.auth.lease_duration * 1000).toISOString();
|
||||
await redis.hSet(`pipeline:${pipelineId}:vault`, {
|
||||
expires_at: newExpiry,
|
||||
last_renewed: new Date().toISOString()
|
||||
});
|
||||
|
||||
broadcastUpdate("token_renewed", {
|
||||
pipeline_id: pipelineId,
|
||||
expires_at: newExpiry
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (e: any) {
|
||||
console.error(`[VAULT] Error renewing token for pipeline ${pipelineId}:`, e.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function revokePipelineToken(pipelineId: string, reason: string): Promise<boolean> {
|
||||
try {
|
||||
const tokenData = await redis.hGetAll(`pipeline:${pipelineId}:vault`);
|
||||
if (!tokenData.accessor) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const initKeys = await Bun.file("/opt/vault/init-keys.json").json();
|
||||
const rootToken = initKeys.root_token;
|
||||
|
||||
// Revoke by accessor
|
||||
const proc = Bun.spawn(["curl", "-sk", "-X", "POST",
|
||||
"-H", `X-Vault-Token: ${rootToken}`,
|
||||
"-d", JSON.stringify({ accessor: tokenData.accessor }),
|
||||
"https://127.0.0.1:8200/v1/auth/token/revoke-accessor"
|
||||
], { stdout: "pipe" });
|
||||
|
||||
await proc.exited;
|
||||
|
||||
// Update Redis
|
||||
await redis.hSet(`pipeline:${pipelineId}:vault`, {
|
||||
status: "revoked",
|
||||
revoked_at: new Date().toISOString(),
|
||||
revoke_reason: reason
|
||||
});
|
||||
|
||||
broadcastUpdate("token_revoked", {
|
||||
pipeline_id: pipelineId,
|
||||
reason: reason,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
|
||||
await appendPipelineLog(pipelineId, "VAULT", `Token revoked: ${reason}`, "WARN");
|
||||
|
||||
return true;
|
||||
} catch (e: any) {
|
||||
console.error(`[VAULT] Error revoking token for pipeline ${pipelineId}:`, e.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function getPipelineTokenStatus(pipelineId: string): Promise<PipelineTokenStatus> {
|
||||
const tokenData = await redis.hGetAll(`pipeline:${pipelineId}:vault`);
|
||||
|
||||
return {
|
||||
pipeline_id: pipelineId,
|
||||
token_active: tokenData.status === "active",
|
||||
issued_at: tokenData.issued_at,
|
||||
expires_at: tokenData.expires_at,
|
||||
last_renewed: tokenData.last_renewed,
|
||||
revoked: tokenData.status === "revoked",
|
||||
revoke_reason: tokenData.revoke_reason
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Error Budget & Observability Integration
|
||||
// =============================================================================
|
||||
|
||||
async function initializeErrorBudget(pipelineId: string): Promise<ErrorBudget> {
|
||||
const budget: ErrorBudget = {
|
||||
pipeline_id: pipelineId,
|
||||
total_errors: 0,
|
||||
errors_per_minute: 0,
|
||||
threshold_exceeded: false,
|
||||
error_types: {}
|
||||
};
|
||||
|
||||
errorBudgets.set(pipelineId, budget);
|
||||
|
||||
await redis.hSet(`pipeline:${pipelineId}:errors`, {
|
||||
total_errors: "0",
|
||||
errors_per_minute: "0",
|
||||
threshold_exceeded: "false",
|
||||
error_types: "{}"
|
||||
});
|
||||
|
||||
return budget;
|
||||
}
|
||||
|
||||
async function recordError(
|
||||
pipelineId: string,
|
||||
errorType: string,
|
||||
severity: "low" | "medium" | "high" | "critical",
|
||||
details: string
|
||||
): Promise<{ threshold_exceeded: boolean; action_taken?: string }> {
|
||||
let budget = errorBudgets.get(pipelineId);
|
||||
if (!budget) {
|
||||
budget = await initializeErrorBudget(pipelineId);
|
||||
}
|
||||
|
||||
budget.total_errors++;
|
||||
budget.error_types[errorType] = (budget.error_types[errorType] || 0) + 1;
|
||||
budget.last_error_at = new Date().toISOString();
|
||||
|
||||
// Calculate errors per minute (rolling window)
|
||||
const errorKey = `pipeline:${pipelineId}:error_times`;
|
||||
const now = Date.now();
|
||||
await redis.rPush(errorKey, String(now));
|
||||
|
||||
// Remove errors older than 1 minute
|
||||
const oneMinuteAgo = now - 60000;
|
||||
const errorTimes = await redis.lRange(errorKey, 0, -1);
|
||||
const recentErrors = errorTimes.filter(t => parseInt(t) > oneMinuteAgo);
|
||||
budget.errors_per_minute = recentErrors.length;
|
||||
|
||||
// Persist to Redis
|
||||
await redis.hSet(`pipeline:${pipelineId}:errors`, {
|
||||
total_errors: String(budget.total_errors),
|
||||
errors_per_minute: String(budget.errors_per_minute),
|
||||
last_error_at: budget.last_error_at,
|
||||
error_types: JSON.stringify(budget.error_types)
|
||||
});
|
||||
|
||||
// Log the error
|
||||
await appendPipelineLog(pipelineId, "ERROR_MONITOR",
|
||||
`Error recorded: ${errorType} (${severity}) - ${details}`,
|
||||
severity === "critical" ? "ERROR" : "WARN"
|
||||
);
|
||||
|
||||
// Check thresholds
|
||||
let actionTaken: string | undefined;
|
||||
|
||||
if (severity === "critical" && ERROR_THRESHOLDS.critical_violation_immediate) {
|
||||
budget.threshold_exceeded = true;
|
||||
actionTaken = "immediate_revocation";
|
||||
await revokePipelineToken(pipelineId, `Critical error: ${errorType}`);
|
||||
await spawnDiagnosticPipeline(pipelineId, errorType, details);
|
||||
} else if (budget.errors_per_minute >= ERROR_THRESHOLDS.max_errors_per_minute) {
|
||||
budget.threshold_exceeded = true;
|
||||
actionTaken = "rate_exceeded_revocation";
|
||||
await revokePipelineToken(pipelineId, `Error rate exceeded: ${budget.errors_per_minute}/min`);
|
||||
await spawnDiagnosticPipeline(pipelineId, "rate_exceeded", `${budget.errors_per_minute} errors in last minute`);
|
||||
} else if (budget.total_errors >= ERROR_THRESHOLDS.max_total_errors) {
|
||||
budget.threshold_exceeded = true;
|
||||
actionTaken = "budget_exhausted_revocation";
|
||||
await revokePipelineToken(pipelineId, `Error budget exhausted: ${budget.total_errors} total errors`);
|
||||
await spawnDiagnosticPipeline(pipelineId, "budget_exhausted", `${budget.total_errors} total errors`);
|
||||
}
|
||||
|
||||
if (budget.threshold_exceeded) {
|
||||
await redis.hSet(`pipeline:${pipelineId}:errors`, "threshold_exceeded", "true");
|
||||
broadcastUpdate("error_threshold", {
|
||||
pipeline_id: pipelineId,
|
||||
total_errors: budget.total_errors,
|
||||
errors_per_minute: budget.errors_per_minute,
|
||||
action_taken: actionTaken
|
||||
});
|
||||
}
|
||||
|
||||
errorBudgets.set(pipelineId, budget);
|
||||
|
||||
return { threshold_exceeded: budget.threshold_exceeded, action_taken: actionTaken };
|
||||
}
|
||||
|
||||
async function spawnDiagnosticPipeline(
|
||||
sourcePipelineId: string,
|
||||
errorType: string,
|
||||
errorDetails: string
|
||||
): Promise<string> {
|
||||
const diagnosticPipelineId = `diagnostic-${sourcePipelineId}-${Date.now().toString(36)}`;
|
||||
|
||||
// Create handoff report
|
||||
const handoffReport = {
|
||||
report_type: "error_handoff",
|
||||
source_pipeline_id: sourcePipelineId,
|
||||
diagnostic_pipeline_id: diagnosticPipelineId,
|
||||
timestamp: new Date().toISOString(),
|
||||
summary: {
|
||||
error_type: errorType,
|
||||
error_details: errorDetails,
|
||||
error_budget: errorBudgets.get(sourcePipelineId)
|
||||
},
|
||||
context: {
|
||||
pipeline_status: await redis.hGetAll(`pipeline:${sourcePipelineId}`),
|
||||
recent_logs: await getPipelineLogs(sourcePipelineId, 20)
|
||||
},
|
||||
recommended_actions: [
|
||||
"Review error patterns",
|
||||
"Check resource availability",
|
||||
"Verify API connectivity",
|
||||
"Consider task decomposition"
|
||||
]
|
||||
};
|
||||
|
||||
// Store handoff report
|
||||
await redis.set(`handoff:${diagnosticPipelineId}`, JSON.stringify(handoffReport));
|
||||
|
||||
// Create diagnostic pipeline entry
|
||||
await redis.hSet(`pipeline:${diagnosticPipelineId}`, {
|
||||
task_id: `diag-task-${Date.now().toString(36)}`,
|
||||
objective: `Diagnose and recover from: ${errorType} in ${sourcePipelineId}`,
|
||||
status: "DIAGNOSTIC",
|
||||
created_at: new Date().toISOString(),
|
||||
source_pipeline: sourcePipelineId,
|
||||
handoff_report: JSON.stringify(handoffReport),
|
||||
agents: JSON.stringify([])
|
||||
});
|
||||
|
||||
await appendPipelineLog(diagnosticPipelineId, "SYSTEM",
|
||||
`Diagnostic pipeline spawned for: ${sourcePipelineId}`, "INFO"
|
||||
);
|
||||
|
||||
broadcastUpdate("diagnostic_spawned", {
|
||||
diagnostic_pipeline_id: diagnosticPipelineId,
|
||||
source_pipeline_id: sourcePipelineId,
|
||||
error_type: errorType,
|
||||
handoff_report: handoffReport
|
||||
});
|
||||
|
||||
return diagnosticPipelineId;
|
||||
}
|
||||
|
||||
async function generateHandoffReport(pipelineId: string): Promise<any> {
|
||||
const pipelineData = await redis.hGetAll(`pipeline:${pipelineId}`);
|
||||
const errorData = await redis.hGetAll(`pipeline:${pipelineId}:errors`);
|
||||
const tokenData = await redis.hGetAll(`pipeline:${pipelineId}:vault`);
|
||||
const logs = await getPipelineLogs(pipelineId, 50);
|
||||
|
||||
return {
|
||||
report_type: "structured_handoff",
|
||||
pipeline_id: pipelineId,
|
||||
generated_at: new Date().toISOString(),
|
||||
pipeline_state: {
|
||||
status: pipelineData.status,
|
||||
created_at: pipelineData.created_at,
|
||||
objective: pipelineData.objective,
|
||||
agents: pipelineData.agents ? JSON.parse(pipelineData.agents) : []
|
||||
},
|
||||
error_summary: {
|
||||
total_errors: parseInt(errorData.total_errors || "0"),
|
||||
errors_per_minute: parseInt(errorData.errors_per_minute || "0"),
|
||||
threshold_exceeded: errorData.threshold_exceeded === "true",
|
||||
error_types: errorData.error_types ? JSON.parse(errorData.error_types) : {}
|
||||
},
|
||||
token_status: {
|
||||
active: tokenData.status === "active",
|
||||
revoked: tokenData.status === "revoked",
|
||||
revoke_reason: tokenData.revoke_reason
|
||||
},
|
||||
recent_activity: logs.slice(0, 20),
|
||||
recommendations: generateRecommendations(pipelineData, errorData)
|
||||
};
|
||||
}
|
||||
|
||||
function generateRecommendations(pipelineData: any, errorData: any): string[] {
|
||||
const recommendations: string[] = [];
|
||||
const totalErrors = parseInt(errorData.total_errors || "0");
|
||||
const errorTypes = errorData.error_types ? JSON.parse(errorData.error_types) : {};
|
||||
|
||||
if (totalErrors > 10) {
|
||||
recommendations.push("Consider breaking down the task into smaller subtasks");
|
||||
}
|
||||
|
||||
if (errorTypes["api_timeout"]) {
|
||||
recommendations.push("Reduce API call frequency or implement backoff");
|
||||
}
|
||||
|
||||
if (errorTypes["validation_failure"]) {
|
||||
recommendations.push("Review input validation rules");
|
||||
}
|
||||
|
||||
if (pipelineData.status === "STUCK" || pipelineData.status === "BLOCKED") {
|
||||
recommendations.push("Check for circular dependencies");
|
||||
recommendations.push("Verify all required resources are available");
|
||||
}
|
||||
|
||||
if (recommendations.length === 0) {
|
||||
recommendations.push("Review logs for specific error patterns");
|
||||
}
|
||||
|
||||
return recommendations;
|
||||
}
|
||||
|
||||
async function getErrorBudget(pipelineId: string): Promise<ErrorBudget | null> {
|
||||
const data = await redis.hGetAll(`pipeline:${pipelineId}:errors`);
|
||||
if (!data.total_errors) return null;
|
||||
|
||||
return {
|
||||
pipeline_id: pipelineId,
|
||||
total_errors: parseInt(data.total_errors),
|
||||
errors_per_minute: parseInt(data.errors_per_minute || "0"),
|
||||
last_error_at: data.last_error_at,
|
||||
threshold_exceeded: data.threshold_exceeded === "true",
|
||||
error_types: data.error_types ? JSON.parse(data.error_types) : {}
|
||||
};
|
||||
}
|
||||
|
||||
// Helper: Determine agent lifecycle state from status
|
||||
function determineAgentLifecycle(pipelineStatus: string, agentState: any): string {
|
||||
if (!agentState) {
|
||||
if (pipelineStatus === "PENDING") return "CREATED";
|
||||
if (pipelineStatus === "COMPLETED") return "SUCCEEDED";
|
||||
if (pipelineStatus === "FAILED" || pipelineStatus === "ERROR") return "ERROR";
|
||||
return "CREATED";
|
||||
}
|
||||
|
||||
const status = agentState.status || pipelineStatus;
|
||||
|
||||
switch (status) {
|
||||
case "PENDING":
|
||||
case "IDLE":
|
||||
return "CREATED";
|
||||
case "WORKING":
|
||||
case "RUNNING":
|
||||
return "BUSY";
|
||||
case "WAITING":
|
||||
case "BLOCKED":
|
||||
return "WAITING";
|
||||
case "COMPLETED":
|
||||
return "SUCCEEDED";
|
||||
case "FAILED":
|
||||
case "ERROR":
|
||||
return "ERROR";
|
||||
default:
|
||||
// Check for handoff
|
||||
if (agentState.handed_off_to) return "HANDED-OFF";
|
||||
return "BUSY";
|
||||
}
|
||||
}
|
||||
|
||||
// Token renewal loop (runs every 30 minutes for active pipelines)
|
||||
async function runTokenRenewalLoop(): Promise<void> {
|
||||
setInterval(async () => {
|
||||
try {
|
||||
const pipelineKeys = await redis.keys("pipeline:*:vault");
|
||||
|
||||
for (const key of pipelineKeys) {
|
||||
const pipelineId = key.replace("pipeline:", "").replace(":vault", "");
|
||||
const tokenData = await redis.hGetAll(key);
|
||||
|
||||
if (tokenData.status === "active" && tokenData.expires_at) {
|
||||
const expiresAt = new Date(tokenData.expires_at).getTime();
|
||||
const now = Date.now();
|
||||
const timeToExpiry = expiresAt - now;
|
||||
|
||||
// Renew if less than 35 minutes to expiry
|
||||
if (timeToExpiry < 35 * 60 * 1000 && timeToExpiry > 0) {
|
||||
console.log(`[VAULT] Renewing token for pipeline ${pipelineId}`);
|
||||
await renewPipelineToken(pipelineId);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error("[VAULT] Token renewal loop error:", e.message);
|
||||
}
|
||||
}, 30 * 60 * 1000); // Every 30 minutes
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Pipeline Spawning
|
||||
// =============================================================================
|
||||
@@ -996,7 +1492,7 @@ interface PipelineConfig {
|
||||
timeout?: number; // Orchestration timeout in seconds (default: 120)
|
||||
}
|
||||
|
||||
async function spawnPipeline(config: PipelineConfig): Promise<{ success: boolean; pipeline_id: string; message: string }> {
|
||||
async function spawnPipeline(config: PipelineConfig): Promise<{ success: boolean; pipeline_id: string; message: string; token_issued?: boolean }> {
|
||||
const pipelineId = `pipeline-${Date.now().toString(36)}`;
|
||||
const taskId = config.task_id || `task-${Date.now().toString(36)}`;
|
||||
|
||||
@@ -1017,6 +1513,19 @@ async function spawnPipeline(config: PipelineConfig): Promise<{ success: boolean
|
||||
// Add to live log
|
||||
await appendPipelineLog(pipelineId, "SYSTEM", `Pipeline ${pipelineId} created for: ${config.objective}`);
|
||||
|
||||
// Issue Vault token for this pipeline
|
||||
await appendPipelineLog(pipelineId, "VAULT", "Requesting pipeline token from Vault...");
|
||||
const tokenInfo = await issuePipelineToken(pipelineId);
|
||||
if (tokenInfo) {
|
||||
await appendPipelineLog(pipelineId, "VAULT", `Token issued (expires: ${new Date(Date.now() + tokenInfo.ttl * 1000).toISOString()})`);
|
||||
} else {
|
||||
await appendPipelineLog(pipelineId, "VAULT", "Token issuance failed - proceeding without dedicated token", "WARN");
|
||||
}
|
||||
|
||||
// Initialize error budget
|
||||
await initializeErrorBudget(pipelineId);
|
||||
await appendPipelineLog(pipelineId, "OBSERVABILITY", "Error budget initialized");
|
||||
|
||||
// Spawn Agent A (Python) and Agent B (Bun) in parallel
|
||||
const agentA = `agent-A-${pipelineId}`;
|
||||
const agentB = `agent-B-${pipelineId}`;
|
||||
@@ -6280,6 +6789,127 @@ const server = Bun.serve({
|
||||
return new Response(JSON.stringify(logs), { headers });
|
||||
}
|
||||
|
||||
// Vault Token Management APIs
|
||||
if (path === "/api/pipeline/token") {
|
||||
const pipelineId = url.searchParams.get("pipeline_id");
|
||||
if (!pipelineId) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
const status = await getPipelineTokenStatus(pipelineId);
|
||||
return new Response(JSON.stringify(status), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/pipeline/token/revoke" && req.method === "POST") {
|
||||
const body = await req.json() as { pipeline_id: string; reason: string };
|
||||
if (!body.pipeline_id || !body.reason) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id and reason required" }), { status: 400, headers });
|
||||
}
|
||||
const success = await revokePipelineToken(body.pipeline_id, body.reason);
|
||||
return new Response(JSON.stringify({ success, message: success ? "Token revoked" : "Failed to revoke token" }), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/pipeline/token/renew" && req.method === "POST") {
|
||||
const body = await req.json() as { pipeline_id: string };
|
||||
if (!body.pipeline_id) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
const success = await renewPipelineToken(body.pipeline_id);
|
||||
return new Response(JSON.stringify({ success, message: success ? "Token renewed" : "Failed to renew token" }), { headers });
|
||||
}
|
||||
|
||||
// Error Budget & Observability APIs
|
||||
if (path === "/api/pipeline/errors") {
|
||||
const pipelineId = url.searchParams.get("pipeline_id");
|
||||
if (!pipelineId) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
const budget = await getErrorBudget(pipelineId);
|
||||
return new Response(JSON.stringify(budget || { pipeline_id: pipelineId, total_errors: 0, errors_per_minute: 0, threshold_exceeded: false, error_types: {} }), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/pipeline/errors/record" && req.method === "POST") {
|
||||
const body = await req.json() as {
|
||||
pipeline_id: string;
|
||||
error_type: string;
|
||||
severity: "low" | "medium" | "high" | "critical";
|
||||
details: string;
|
||||
};
|
||||
if (!body.pipeline_id || !body.error_type || !body.severity) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id, error_type, and severity required" }), { status: 400, headers });
|
||||
}
|
||||
const result = await recordError(body.pipeline_id, body.error_type, body.severity, body.details || "");
|
||||
return new Response(JSON.stringify(result), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/observability/handoff" && req.method === "POST") {
|
||||
const body = await req.json() as { pipeline_id: string };
|
||||
if (!body.pipeline_id) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
const report = await generateHandoffReport(body.pipeline_id);
|
||||
return new Response(JSON.stringify(report), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/observability/diagnostic" && req.method === "POST") {
|
||||
const body = await req.json() as { pipeline_id: string; error_type: string; details: string };
|
||||
if (!body.pipeline_id || !body.error_type) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id and error_type required" }), { status: 400, headers });
|
||||
}
|
||||
const diagnosticId = await spawnDiagnosticPipeline(body.pipeline_id, body.error_type, body.details || "");
|
||||
return new Response(JSON.stringify({ success: true, diagnostic_pipeline_id: diagnosticId }), { headers });
|
||||
}
|
||||
|
||||
if (path === "/api/pipeline/metrics") {
|
||||
const pipelineId = url.searchParams.get("pipeline_id");
|
||||
if (!pipelineId) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
// Get metrics from multi-agent coordination
|
||||
const metricsKey = `metrics:${pipelineId}`;
|
||||
const metricsData = await redis.hGetAll(metricsKey);
|
||||
const errorBudget = await getErrorBudget(pipelineId);
|
||||
const tokenStatus = await getPipelineTokenStatus(pipelineId);
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
pipeline_id: pipelineId,
|
||||
coordination: metricsData,
|
||||
error_budget: errorBudget,
|
||||
token_status: tokenStatus
|
||||
}), { headers });
|
||||
}
|
||||
|
||||
// Agent Lifecycle Status API
|
||||
if (path === "/api/agents/lifecycle") {
|
||||
const pipelineId = url.searchParams.get("pipeline_id");
|
||||
if (!pipelineId) {
|
||||
return new Response(JSON.stringify({ error: "pipeline_id required" }), { status: 400, headers });
|
||||
}
|
||||
|
||||
// Get agents from pipeline
|
||||
const pipelineKey = `pipeline:${pipelineId}`;
|
||||
const agentsRaw = await redis.hGet(pipelineKey, "agents");
|
||||
const agents = agentsRaw ? JSON.parse(agentsRaw) : [];
|
||||
|
||||
// Enrich with state from multi-agent coordination
|
||||
const enrichedAgents = [];
|
||||
for (const agent of agents) {
|
||||
const stateKey = `agents:${pipelineId}`;
|
||||
const stateData = await redis.hGet(stateKey, agent.type);
|
||||
let state = null;
|
||||
if (stateData) {
|
||||
try { state = JSON.parse(stateData); } catch {}
|
||||
}
|
||||
|
||||
enrichedAgents.push({
|
||||
...agent,
|
||||
lifecycle: determineAgentLifecycle(agent.status, state),
|
||||
state: state
|
||||
});
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify({ pipeline_id: pipelineId, agents: enrichedAgents }), { headers });
|
||||
}
|
||||
|
||||
// Plan Execution APIs
|
||||
if (path === "/api/plans") {
|
||||
const pipelineId = url.searchParams.get("pipeline_id");
|
||||
@@ -6758,6 +7388,10 @@ async function main() {
|
||||
|
||||
await connectRedis();
|
||||
|
||||
// Start Vault token renewal loop for active pipelines
|
||||
runTokenRenewalLoop();
|
||||
console.log("[VAULT] Token renewal loop started");
|
||||
|
||||
console.log(`\n[SERVER] Dashboard running at http://localhost:${PORT}`);
|
||||
console.log("[SERVER] WebSocket endpoint: ws://localhost:" + PORT + "/ws");
|
||||
console.log("[SERVER] Press Ctrl+C to stop\n");
|
||||
|
||||
Reference in New Issue
Block a user