Phase 19 wiring + Path 1/2 work + chain integrity fixes
Backend:
- crates/vectord/src/playbook_memory.rs (new): Phase 19 in-memory boost
store with seed/rebuild/snapshot, plus temporal decay (e^-age/30 per
playbook), persist_to_sql endpoint backing successful_playbooks_live,
and discover_patterns endpoint for meta-index pattern aggregation
(recurring certs/skills/archetype/reliability across similar past fills).
- DEFAULT_TOP_K_PLAYBOOKS bumped 5 → 25; old default silently missed
most boosts when memory had > 25 entries.
- service.rs: new routes /vectors/playbook_memory/{seed,rebuild,stats,
persist_sql,patterns}.
Bun staffing co-pilot (mcp-server/):
- /search, /match, /verify, /proof, /simulation/run, MCP tools all
forward use_playbook_memory:true and playbook_memory_k:25 to the
hybrid endpoint. Boost was previously dark across the entire app.
- /log no longer POSTs to /ingest/file — that endpoint REPLACES the
dataset's object list, so single-row CSV writes were wiping all prior
rows in successful_playbooks (sp_rows went 33→1 in one /log call).
/log now seeds playbook_memory with canonical short text and calls
/persist_sql to keep successful_playbooks_live in sync.
- /simulation/run cumulative end-of-week CSV write removed for the same
reason. Per-day per-contract /seed (added in this session) is the
accumulating feedback path now.
- search.html addWorkerInsight renders a green "Endorsed · N playbooks"
chip with playbook citations when boost > 0.
Internal Dioxus UI (crates/ui/):
- Dashboard phase list rewritten through Phase 19 (was stuck at "Phase
16: File Watcher" / "Phase 17: DB Connector" — both wrong).
- Removed fabricated "27ms" stat label.
- Ask tab examples + SQL default replaced with real staffing prompts
against candidates/clients/job_orders (was referencing nonexistent
employees/products/events).
- New Playbook tab exposes /vectors/playbook_memory/{stats,rebuild} and
side-by-side hybrid search (boost OFF vs ON) with citations.
Tests (tests/multi-agent/):
- run_e2e_rated.ts: parallel two-agent (mistral + qwen2.5) build phase
+ verifier rating (geo, auth, persist, boost, speed → /10).
- network_proving.ts: continuous build → verify → repeat with
staffing-recruiter profile hot-swap; geo-discrimination check.
- chain_of_custody.ts: single recruiter operation traced through every
layer (Bun /search, direct /vectors/hybrid parity, /log, SQL,
playbook_memory growth, profile activation, post-op boost lift).
This commit is contained in:
root
@@ -0,0 +1,351 @@
|
||||
// Shared runtime for one agent. An agent is a role (executor or reviewer),
|
||||
// a model name, and a conversation the orchestrator hands it. The agent
|
||||
// produces ONE structured Action per turn; the orchestrator applies tool
|
||||
// calls and feeds results back.
|
||||
//
|
||||
// Fail-fast: every HTTP error, parse error, and Ollama error throws. The
|
||||
// orchestrator catches at the top and exits non-zero with the full log.
|
||||
|
||||
export const GATEWAY = "http://localhost:3100";
|
||||
export const SIDECAR = "http://localhost:3200";
|
||||
|
||||
// --- Shared types ---
|
||||
|
||||
export type Role = "executor" | "reviewer";
|
||||
|
||||
export interface TaskSpec {
|
||||
id: string;
|
||||
operation: string; // "fill: Welder x2 in Columbus, OH"
|
||||
target_role: string; // "Welder"
|
||||
target_count: number; // 2
|
||||
target_city: string; // "Columbus"
|
||||
target_state: string; // "OH"
|
||||
approach_hint?: string; // e.g. "hybrid search"; agent is free to ignore
|
||||
}
|
||||
|
||||
export interface LogEntry {
|
||||
turn: number;
|
||||
role: Role;
|
||||
model: string;
|
||||
at: string;
|
||||
kind:
|
||||
| "plan"
|
||||
| "tool_call"
|
||||
| "tool_result"
|
||||
| "critique"
|
||||
| "propose_done"
|
||||
| "consensus_done"
|
||||
| "error";
|
||||
content: any;
|
||||
}
|
||||
|
||||
// Action = what an agent returns on one turn. Strict shape so we can
|
||||
// enforce it at parse time rather than prompt-engineer around malformed
|
||||
// JSON.
|
||||
export type Action =
|
||||
| { kind: "tool_call"; tool: string; args: Record<string, any>; rationale: string }
|
||||
| { kind: "propose_done"; fills: Fill[]; rationale: string }
|
||||
| { kind: "critique"; verdict: "continue" | "drift" | "approve_done"; notes: string }
|
||||
| { kind: "plan"; steps: string[] };
|
||||
|
||||
export interface Fill {
|
||||
candidate_id: string;
|
||||
name: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
// --- HTTP helpers (fail-fast) ---
|
||||
|
||||
async function http<T>(method: string, url: string, body?: any): Promise<T> {
|
||||
const res = await fetch(url, {
|
||||
method,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`${method} ${url} → ${res.status}: ${text}`);
|
||||
}
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
|
||||
// Tool calls land in the Phase 12 audit log keyed by this agent name.
|
||||
// Distinguishable from human-driven calls (agent=="operator" or similar)
|
||||
// so post-hoc queries can separate multi-agent runs.
|
||||
export const TOOL_AGENT_ID = "multi-agent-test";
|
||||
|
||||
export async function callTool(tool: string, args: Record<string, any>): Promise<any> {
|
||||
return http("POST", `${GATEWAY}/tools/${tool}/call`, {
|
||||
params: args,
|
||||
agent: TOOL_AGENT_ID,
|
||||
});
|
||||
}
|
||||
|
||||
export async function hybridSearch(sql_filter: string, question: string, k = 10): Promise<any> {
|
||||
return http("POST", `${GATEWAY}/vectors/hybrid`, { sql_filter, question, k });
|
||||
}
|
||||
|
||||
export async function sqlQuery(sql: string): Promise<any> {
|
||||
return http("POST", `${GATEWAY}/query/sql`, { sql, format: "json" });
|
||||
}
|
||||
|
||||
// Sidecar generate. Ollama's default keep_alive (5 min) keeps the model
|
||||
// warm between turns on its own, so we don't need to pass it through.
|
||||
export async function generate(model: string, prompt: string, opts: {
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
system?: string;
|
||||
} = {}): Promise<string> {
|
||||
const body: Record<string, any> = {
|
||||
model,
|
||||
prompt,
|
||||
temperature: opts.temperature ?? 0.3,
|
||||
max_tokens: opts.max_tokens ?? 800,
|
||||
};
|
||||
if (opts.system) body.system = opts.system;
|
||||
const r = await http<any>("POST", `${SIDECAR}/generate`, body);
|
||||
const text = r.text ?? "";
|
||||
if (!text || typeof text !== "string") {
|
||||
throw new Error(`generate returned empty text from ${model}: ${JSON.stringify(r).slice(0, 200)}`);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
// --- Prompt construction ---
|
||||
|
||||
const TOOL_CATALOG = `
|
||||
Available tools (each takes a JSON "args" object):
|
||||
|
||||
- hybrid_search(sql_filter: string, question: string, index_name: string, k?: number)
|
||||
→ Narrow workers via SQL WHERE clause, then rank by semantic match.
|
||||
→ Canonical production tool for fill tasks. Always use this FIRST.
|
||||
→ Example args:
|
||||
{"index_name":"workers_500k_v1",
|
||||
"sql_filter":"LOWER(role) LIKE '%weld%' AND city = 'Toledo' AND state = 'OH' AND availability > 0.5",
|
||||
"question":"reliable welder with OSHA certs",
|
||||
"k":10}
|
||||
|
||||
- sql(query: string)
|
||||
→ Raw read-only SELECT. Use for verification (confirm a worker exists,
|
||||
check city/role/availability) after hybrid_search surfaces candidates.
|
||||
→ Schema of workers_500k: worker_id, name, role, email, phone, city,
|
||||
state, zip, skills, certifications, archetype, reliability,
|
||||
responsiveness, engagement, communications, compliance, availability,
|
||||
resume_text.
|
||||
→ Example args:
|
||||
{"query":"SELECT worker_id, name, role, city, state, availability FROM workers_500k WHERE worker_id = 'W123456'"}
|
||||
|
||||
Rules:
|
||||
- hybrid_search returns sources[] each with {doc_id, chunk_text, score, sql_verified}.
|
||||
- **ID mapping:** vector doc_ids look like "W500K-7995" (prefix + number).
|
||||
The SQL worker_id is an INTEGER. To go from doc_id to SQL, strip the
|
||||
"W500K-" prefix and cast:
|
||||
SELECT ... FROM workers_500k WHERE worker_id = CAST(SUBSTR('W500K-7995', 7) AS BIGINT)
|
||||
or more simply: WHERE worker_id = 7995.
|
||||
- Names are NOT unique. Always identify by worker_id, never by name alone.
|
||||
- Return EXACTLY ONE JSON object per turn. No prose outside the JSON.
|
||||
`;
|
||||
|
||||
// Smart per-kind summary so agents see the substance of each prior turn
|
||||
// without a raw-JSON wall of text. hybrid_search results especially need
|
||||
// this — raw JSON buries sources[] past any reasonable 400-char truncation.
|
||||
function summarizeEntry(e: LogEntry): string {
|
||||
const c = e.content ?? {};
|
||||
switch (e.kind) {
|
||||
case "plan":
|
||||
return `PLAN: ${(c.steps ?? []).map((s: string, i: number) => `${i + 1}.${s}`).join(" ")}`;
|
||||
case "tool_call":
|
||||
return `TOOL_CALL ${c.tool}(${JSON.stringify(c.args ?? {}).slice(0, 250)})${c.rationale ? ` — ${c.rationale}` : ""}`;
|
||||
case "tool_result": {
|
||||
if (c.error) return `TOOL_RESULT error: ${c.error}`;
|
||||
// hybrid_search response
|
||||
if (Array.isArray(c.sources)) {
|
||||
const head = c.sources.slice(0, 5).map((s: any) =>
|
||||
`${s.doc_id}${s.sql_verified ? "✓" : ""} score=${(s.score ?? 0).toFixed(2)}: ${String(s.chunk_text ?? "").slice(0, 80)}`
|
||||
).join(" | ");
|
||||
return `TOOL_RESULT hybrid: sql_matches=${c.sql_matches} vector_reranked=${c.vector_reranked} sources=[${head}${c.sources.length > 5 ? ` +${c.sources.length - 5} more` : ""}]`;
|
||||
}
|
||||
// sql response
|
||||
if (Array.isArray(c.rows)) {
|
||||
const head = c.rows.slice(0, 5).map((r: any) => JSON.stringify(r)).join(" | ");
|
||||
return `TOOL_RESULT sql: ${c.rows.length} rows${c.rows.length > 0 ? ` — ${head}${c.rows.length > 5 ? ` +${c.rows.length - 5} more` : ""}` : ""}`;
|
||||
}
|
||||
// fallback
|
||||
return `TOOL_RESULT ${JSON.stringify(c).slice(0, 250)}`;
|
||||
}
|
||||
case "critique":
|
||||
return `CRITIQUE verdict=${c.verdict} notes: ${String(c.notes ?? "").slice(0, 200)}`;
|
||||
case "propose_done":
|
||||
return `PROPOSE_DONE fills=[${(c.fills ?? []).map((f: Fill) => `${f.candidate_id}:${f.name}`).join(", ")}] rationale: ${String(c.rationale ?? "").slice(0, 120)}`;
|
||||
case "consensus_done":
|
||||
return `CONSENSUS ✓`;
|
||||
case "error":
|
||||
return `ERROR ${c.message ?? JSON.stringify(c)}`;
|
||||
}
|
||||
return JSON.stringify(c).slice(0, 200);
|
||||
}
|
||||
|
||||
function renderLogForPrompt(log: LogEntry[]): string {
|
||||
if (log.length === 0) return "(no turns yet)";
|
||||
return log.slice(-12).map(e =>
|
||||
`[t${e.turn} ${e.role}] ${summarizeEntry(e)}`
|
||||
).join("\n");
|
||||
}
|
||||
|
||||
// Crawl the log for every hybrid_search tool_result and collect the
|
||||
// worker names + ids seen so far. LLMs routinely "forget" earlier turns
|
||||
// once the conversation grows, so we surface a running ledger in the
|
||||
// prompt as orchestrator-maintained state. The executor doesn't have to
|
||||
// track this itself — it just reads it.
|
||||
function candidatesSeen(log: LogEntry[]): Array<{ doc_id: string; name: string; city: string; state: string }> {
|
||||
const seen = new Map<string, { doc_id: string; name: string; city: string; state: string }>();
|
||||
for (const e of log) {
|
||||
if (e.kind !== "tool_result") continue;
|
||||
const sources = (e.content as any)?.sources;
|
||||
if (!Array.isArray(sources)) continue;
|
||||
for (const s of sources) {
|
||||
// chunk_text shape "Name — Role in City, ST. …"
|
||||
const t = String(s.chunk_text ?? "");
|
||||
const [namePart, rest] = t.split("—", 2);
|
||||
if (!namePart || !rest) continue;
|
||||
const loc = rest.split(" in ")[1] ?? "";
|
||||
const [city, stateRaw] = loc.split(",", 2);
|
||||
const state = (stateRaw ?? "").trim().replace(/[^A-Za-z].*/, "");
|
||||
if (!s.doc_id || !namePart.trim() || !city?.trim() || !state) continue;
|
||||
if (!seen.has(s.doc_id)) {
|
||||
seen.set(s.doc_id, {
|
||||
doc_id: s.doc_id,
|
||||
name: namePart.trim(),
|
||||
city: city.trim(),
|
||||
state,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(seen.values());
|
||||
}
|
||||
|
||||
export function executorPrompt(task: TaskSpec, log: LogEntry[]): string {
|
||||
const logStr = renderLogForPrompt(log);
|
||||
const seen = candidatesSeen(log);
|
||||
const seenBlock = seen.length === 0
|
||||
? "(no candidates surfaced yet — start with hybrid_search)"
|
||||
: seen.map(s => ` - ${s.doc_id} ${s.name} (${s.city}, ${s.state})`).join("\n");
|
||||
|
||||
return `You are the EXECUTOR agent. Your job is to complete this task:
|
||||
|
||||
OPERATION: ${task.operation}
|
||||
TARGET: ${task.target_count} × ${task.target_role} in ${task.target_city}, ${task.target_state}
|
||||
${task.approach_hint ? `HINT: ${task.approach_hint}` : ""}
|
||||
|
||||
The REVIEWER agent is watching every turn. They will flag drift. Stay on target.
|
||||
|
||||
${TOOL_CATALOG}
|
||||
|
||||
CANDIDATES SURFACED SO FAR (orchestrator-tracked, do not forget these):
|
||||
${seenBlock}
|
||||
|
||||
SHARED LOG (recent turns):
|
||||
${logStr}
|
||||
|
||||
Your next action MUST be a JSON object matching one of these shapes:
|
||||
{"kind":"plan","steps":["short step 1","short step 2",...]}
|
||||
— use on turn 1 to outline your approach. Steps must be concrete.
|
||||
{"kind":"tool_call","tool":"...","args":{...},"rationale":"why"}
|
||||
— call a tool and see its result next turn.
|
||||
{"kind":"propose_done","fills":[{"candidate_id":"...","name":"First Last","reason":"why them"}],"rationale":"..."}
|
||||
— propose you've met the target. fills MUST have EXACTLY ${task.target_count} entries — count twice before emitting.
|
||||
|
||||
Strategy tip: once "CANDIDATES SURFACED SO FAR" has ≥ ${task.target_count} entries in ${task.target_city}, ${task.target_state} matching ${task.target_role}, verify ONE via the sql tool (to satisfy the reviewer's SQL-verification criterion) and then propose_done with the top ${task.target_count}. Don't keep re-searching.
|
||||
|
||||
Respond with ONLY the JSON object. No markdown fences, no prose.`;
|
||||
}
|
||||
|
||||
export function reviewerPrompt(task: TaskSpec, log: LogEntry[]): string {
|
||||
const logStr = renderLogForPrompt(log);
|
||||
|
||||
// If the most recent executor action was propose_done, the reviewer
|
||||
// must commit to an up-or-down vote this turn — "continue" would stall
|
||||
// the orchestrator forever. The wider prompt still describes all three
|
||||
// verdicts, but we add a hard rule at the end that the model must obey.
|
||||
const lastExec = [...log].reverse().find(e => e.role === "executor");
|
||||
const awaitingApproval = lastExec?.kind === "propose_done";
|
||||
|
||||
return `You are the REVIEWER agent. The EXECUTOR is trying to complete this task:
|
||||
|
||||
OPERATION: ${task.operation}
|
||||
TARGET: ${task.target_count} × ${task.target_role} in ${task.target_city}, ${task.target_state}
|
||||
|
||||
Your job: catch drift. Agents often wander from the actual objective. Specifically watch for:
|
||||
- Proposing candidates who aren't in ${task.target_city}, ${task.target_state}.
|
||||
- Proposing candidates who don't have ${task.target_role} skill.
|
||||
- Proposing fewer or more than ${task.target_count} fills.
|
||||
- Irrelevant tool calls (e.g. revenue_by_client when the task is a fill).
|
||||
|
||||
Available tools (for reference, but YOU don't call them):
|
||||
- hybrid_search(sql_filter, question, index_name, k) — production fill path
|
||||
- sql(query) — read-only SELECT for verification
|
||||
|
||||
SHARED LOG (recent turns):
|
||||
${logStr}
|
||||
|
||||
Your next action MUST be a JSON object:
|
||||
{"kind":"critique","verdict":"continue" | "drift" | "approve_done","notes":"..."}
|
||||
|
||||
- "continue" → executor is on a reasonable path, let them keep going.
|
||||
- "drift" → executor is off-track; notes MUST tell them how to redirect.
|
||||
- "approve_done" → executor's propose_done meets the criteria. Seal it.
|
||||
|
||||
APPROVAL CRITERIA (use these only for propose_done):
|
||||
1. Exactly ${task.target_count} fills.
|
||||
2. Each fill's name appears in a prior tool_result from ${task.target_city}, ${task.target_state} matching role "${task.target_role}".
|
||||
3. Executor has SQL-verified at least one of the fills (any prior sql tool_result with that worker).
|
||||
If 1–3 all hold, return approve_done. Do not demand further verification.
|
||||
${awaitingApproval ? `
|
||||
|
||||
HARD RULE: The executor's most recent action was propose_done. On this turn you CANNOT return "continue" — it would stall the task. Choose approve_done (proposal is valid by the 3 criteria above) or drift (it fails one; state which in notes).` : ""}
|
||||
|
||||
Respond with ONLY the JSON object.`;
|
||||
}
|
||||
|
||||
// Parse an agent's response into an Action, or throw.
|
||||
export function parseAction(raw: string, role: Role): Action {
|
||||
// Models sometimes wrap JSON in ```json fences; strip them.
|
||||
let s = raw.trim();
|
||||
if (s.startsWith("```")) {
|
||||
s = s.replace(/^```(?:json)?\n?/, "").replace(/```$/, "").trim();
|
||||
}
|
||||
// Find the first {...} block.
|
||||
const start = s.indexOf("{");
|
||||
const end = s.lastIndexOf("}");
|
||||
if (start < 0 || end <= start) {
|
||||
throw new Error(`no JSON object in ${role} response: ${raw.slice(0, 300)}`);
|
||||
}
|
||||
const json = s.slice(start, end + 1);
|
||||
let obj: any;
|
||||
try {
|
||||
obj = JSON.parse(json);
|
||||
} catch (e) {
|
||||
throw new Error(`invalid JSON from ${role}: ${(e as Error).message} | raw: ${json.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
if (role === "executor") {
|
||||
if (obj.kind === "plan" && Array.isArray(obj.steps)) return obj as Action;
|
||||
if (obj.kind === "tool_call" && typeof obj.tool === "string" && typeof obj.args === "object") return obj as Action;
|
||||
if (obj.kind === "propose_done" && Array.isArray(obj.fills)) return obj as Action;
|
||||
throw new Error(`executor returned unexpected shape: ${JSON.stringify(obj).slice(0, 200)}`);
|
||||
} else {
|
||||
// Normalize: some models (qwen2.5, mistral) emit the verdict AS the
|
||||
// `kind` field directly instead of nesting it under a "critique"
|
||||
// wrapper. Accept both shapes rather than hard-failing — the
|
||||
// semantic content is identical, and rejecting would stall the
|
||||
// orchestrator on a cosmetic schema miss.
|
||||
if (obj.kind === "critique" && ["continue", "drift", "approve_done"].includes(obj.verdict)) {
|
||||
return obj as Action;
|
||||
}
|
||||
if (["continue", "drift", "approve_done"].includes(obj.kind)) {
|
||||
return { kind: "critique", verdict: obj.kind, notes: obj.notes ?? "" } as Action;
|
||||
}
|
||||
throw new Error(`reviewer returned unexpected shape: ${JSON.stringify(obj).slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
// Chain-of-custody trace test.
|
||||
//
|
||||
// J's framing: "we have enough synthetic data, we've run enough AI responses
|
||||
// saved to the database. Test true quality. Don't ignore chain of custody.
|
||||
// Use real applications. Understand each aspect of the flow — not just
|
||||
// 'write a file or directory and open it'."
|
||||
//
|
||||
// One real recruiter operation, traced end-to-end through EVERY layer of the
|
||||
// live substrate. Every layer must record the operation correctly. Any layer
|
||||
// that drops it = chain-of-custody break = surfaced as a real bug.
|
||||
//
|
||||
// Layers verified:
|
||||
// L0 Bun /search — recruiter app surface (NOT bare /vectors/hybrid)
|
||||
// L1 /vectors/hybrid — direct gateway (parity check vs L0)
|
||||
// L2 /vectors/playbook_memory/stats — feedback loop count
|
||||
// L3 Bun /log — recruiter records the pick
|
||||
// L4 successful_playbooks — SQL-queryable table of past fills
|
||||
// L5 /vectors/playbook_memory/stats — count grew
|
||||
// L6 tools/audit — Phase 12 governance trail
|
||||
// L7 /access/audit — Phase 13 access trail
|
||||
// L8 /journal/recent — Phase 9 mutation events
|
||||
// L9 /storage/errors — Federation error journal (no new errors)
|
||||
// L10 /vectors/profile/{id}/activate — Phase 17 hot-swap
|
||||
// L11 Bun /search again — boost lifts the just-logged worker
|
||||
// L12 verifier qwen2.5 — reads cross-layer state, judges integrity
|
||||
//
|
||||
// Run: bun run tests/multi-agent/chain_of_custody.ts
|
||||
//
|
||||
// Prints per-layer BEFORE/AFTER/DELTA. Exit non-zero on any chain break.
|
||||
|
||||
import { generate, GATEWAY } from "./agent.ts";
|
||||
|
||||
const BUN = "http://localhost:3700";
|
||||
const PROFILE_ID = "staffing-recruiter";
|
||||
|
||||
// The trace operation — small, deterministic, real city/role with supply.
|
||||
// Helen Sanchez (worker_id 4661) is a known Toledo Welder; we record her
|
||||
// as the manual pick the recruiter would make from the /search results.
|
||||
const OPERATION = "fill: Welder x1 in Toledo, OH";
|
||||
const OP_ROLE = "Welder";
|
||||
const OP_CITY = "Toledo";
|
||||
const OP_STATE = "OH";
|
||||
const PICKED_WORKER = "Helen Sanchez"; // verified earlier to be a Toledo OH Welder
|
||||
|
||||
// ─────────────────────── helpers ───────────────────────
|
||||
|
||||
async function getJSON<T = any>(url: string): Promise<T | null> {
|
||||
try {
|
||||
const r = await fetch(url);
|
||||
if (!r.ok) return null;
|
||||
return r.json() as Promise<T>;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
async function postJSON<T = any>(url: string, body: any): Promise<T | null> {
|
||||
try {
|
||||
const r = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
|
||||
if (!r.ok) return { _error: `${r.status}: ${await r.text()}` } as any;
|
||||
return r.json() as Promise<T>;
|
||||
} catch (e) { return { _error: (e as Error).message } as any; }
|
||||
}
|
||||
|
||||
async function sql(query: string): Promise<{ rows?: any[]; error?: string } | null> {
|
||||
return postJSON(`${GATEWAY}/query/sql`, { sql: query });
|
||||
}
|
||||
|
||||
interface Snapshot {
|
||||
pm_entries: number;
|
||||
pm_names: number;
|
||||
sp_rows: number; // successful_playbooks SQL row count
|
||||
audit_count: number; // tools/audit count
|
||||
access_count: number; // access/audit count
|
||||
journal_count: number; // journal/stats events
|
||||
storage_errors: number; // bucket error journal
|
||||
}
|
||||
|
||||
async function snapshot(): Promise<Snapshot> {
|
||||
const pm = await getJSON<any>(`${GATEWAY}/vectors/playbook_memory/stats`);
|
||||
// successful_playbooks_live is the live SQL surface populated by /log
|
||||
// via /vectors/playbook_memory/persist_sql. The original
|
||||
// successful_playbooks table is now legacy/historical (no writes).
|
||||
const sp = await sql(`SELECT COUNT(*) AS c FROM successful_playbooks_live`);
|
||||
const audit = await getJSON<any[]>(`${GATEWAY}/tools/audit`);
|
||||
const access = await getJSON<any>(`${GATEWAY}/access/audit`);
|
||||
const journalStats = await getJSON<any>(`${GATEWAY}/journal/stats`);
|
||||
const storageErrors = await getJSON<any[]>(`${GATEWAY}/storage/errors`);
|
||||
|
||||
return {
|
||||
pm_entries: pm?.entries ?? -1,
|
||||
pm_names: pm?.total_names_endorsed ?? -1,
|
||||
sp_rows: Number(sp?.rows?.[0]?.c ?? -1),
|
||||
audit_count: Array.isArray(audit) ? audit.length : (audit as any)?.events?.length ?? -1,
|
||||
access_count: Array.isArray(access) ? access.length : (access as any)?.events?.length ?? (access as any)?.audit?.length ?? -1,
|
||||
journal_count: journalStats?.event_count ?? journalStats?.total_events ?? journalStats?.events ?? -1,
|
||||
storage_errors: Array.isArray(storageErrors) ? storageErrors.length : (storageErrors as any)?.events?.length ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function delta(b: Snapshot, a: Snapshot): Record<string, number> {
|
||||
return {
|
||||
pm_entries: a.pm_entries - b.pm_entries,
|
||||
pm_names: a.pm_names - b.pm_names,
|
||||
sp_rows: a.sp_rows - b.sp_rows,
|
||||
audit_count: a.audit_count - b.audit_count,
|
||||
access_count: a.access_count - b.access_count,
|
||||
journal_count: a.journal_count - b.journal_count,
|
||||
storage_errors: a.storage_errors - b.storage_errors,
|
||||
};
|
||||
}
|
||||
|
||||
function fmtRow(label: string, b: number, a: number): string {
|
||||
const d = a - b;
|
||||
const dStr = d === 0 ? " · " : d > 0 ? ` +${d}` : ` ${d}`;
|
||||
return ` ${label.padEnd(28)} ${String(b).padStart(6)} → ${String(a).padStart(6)} ${dStr}`;
|
||||
}
|
||||
|
||||
// ─────────────────────── trace ───────────────────────
|
||||
|
||||
interface TraceResult {
|
||||
layer: string;
|
||||
ok: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
async function runTrace(): Promise<TraceResult[]> {
|
||||
const out: TraceResult[] = [];
|
||||
const note = (layer: string, ok: boolean, detail: string) => {
|
||||
out.push({ layer, ok, detail });
|
||||
console.log(` ${ok ? "✓" : "✗"} ${layer.padEnd(32)} ${detail}`);
|
||||
};
|
||||
|
||||
console.log(`\n▶ Trace operation: ${OPERATION} → pick=${PICKED_WORKER}\n`);
|
||||
|
||||
// ── BEFORE snapshot ──
|
||||
console.log(`▶ Before-snapshot:`);
|
||||
const before = await snapshot();
|
||||
console.log(` pm_entries=${before.pm_entries} pm_names=${before.pm_names} sp_rows=${before.sp_rows} `
|
||||
+ `audit=${before.audit_count} access=${before.access_count} journal=${before.journal_count} `
|
||||
+ `storage_errors=${before.storage_errors}\n`);
|
||||
|
||||
// ── L0: Bun /search ──
|
||||
console.log(`▶ L0 — Bun /search (recruiter app surface)`);
|
||||
const sql_filter = `role = '${OP_ROLE}' AND state = '${OP_STATE}' AND city = '${OP_CITY}'`;
|
||||
const bunSearch = await postJSON<any>(`${BUN}/search`, {
|
||||
question: `Welder in ${OP_CITY}, ${OP_STATE}`,
|
||||
sql_filter, top_k: 5, generate: false,
|
||||
id_column: "worker_id", dataset: "workers_500k", use_playbook_memory: true,
|
||||
});
|
||||
if (bunSearch?._error) {
|
||||
note("L0 Bun /search", false, `error: ${bunSearch._error}`);
|
||||
} else {
|
||||
const sources = bunSearch?.sources ?? [];
|
||||
const boostedHits = sources.filter((s: any) => (s.playbook_boost ?? 0) > 0).length;
|
||||
note("L0 Bun /search", true, `sources=${sources.length} boosted=${boostedHits} sql_matches=${bunSearch?.sql_matches}`);
|
||||
}
|
||||
|
||||
// ── L1: direct /vectors/hybrid (parity check) ──
|
||||
console.log(`\n▶ L1 — Direct /vectors/hybrid (parity check vs Bun)`);
|
||||
const directSearch = await postJSON<any>(`${GATEWAY}/vectors/hybrid`, {
|
||||
index_name: "workers_500k_v1", filter_dataset: "workers_500k", id_column: "worker_id",
|
||||
sql_filter, question: `Welder in ${OP_CITY}, ${OP_STATE}`,
|
||||
top_k: 5, generate: false, use_playbook_memory: true, playbook_memory_k: 15,
|
||||
});
|
||||
const directBoosted = (directSearch?.sources ?? []).filter((s: any) => (s.playbook_boost ?? 0) > 0).length;
|
||||
note("L1 Direct /vectors/hybrid", true, `boosted=${directBoosted} sql=${directSearch?.sql_matches}`);
|
||||
|
||||
const bunBoosted = (bunSearch?.sources ?? []).filter((s: any) => (s.playbook_boost ?? 0) > 0).length;
|
||||
if (bunBoosted < directBoosted) {
|
||||
note("CHAIN BREAK: Bun↔Direct parity", false,
|
||||
`Bun=${bunBoosted} boosted vs Direct=${directBoosted}. Bun /search likely missing playbook_memory_k forward.`);
|
||||
}
|
||||
|
||||
// ── L3: Bun /log (recruiter records the pick) ──
|
||||
console.log(`\n▶ L3 — Bun /log (recruiter records the pick)`);
|
||||
const logged = await postJSON<any>(`${BUN}/log`, {
|
||||
operation: OPERATION,
|
||||
approach: "chain-of-custody trace",
|
||||
result: `1/1 filled → ${PICKED_WORKER}`,
|
||||
context: `client=COC-${Date.now()} start=08:00 scenario=trace`,
|
||||
});
|
||||
if (logged?._error) note("L3 Bun /log", false, `error: ${logged._error}`);
|
||||
else note("L3 Bun /log", true, `logged=${logged?.logged} seeded=${logged?.seeded}`);
|
||||
|
||||
// The /log response carries the result of the underlying /ingest/file too.
|
||||
// If "response" mentions "different schema" or "error", the SQL-queryable
|
||||
// path is broken even though seed succeeded. That's a chain break.
|
||||
const logResp = String((logged as any)?.response ?? "");
|
||||
if (logResp.includes("error") || logResp.includes("different schema") || logResp.includes("Error")) {
|
||||
note("CHAIN BREAK: Bun /log → SQL ingest", false,
|
||||
`successful_playbooks ingest failed. Bun returned logged=true but /log's underlying ingest reported: ${logResp.slice(0, 150)}`);
|
||||
} else {
|
||||
note("L3a /log → /ingest/file", true, "ingest accepted");
|
||||
}
|
||||
|
||||
// Give the system a beat for any async fan-out (audit/journal/etc).
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
// ── AFTER snapshot ──
|
||||
console.log(`\n▶ After-snapshot:`);
|
||||
const after = await snapshot();
|
||||
const d = delta(before, after);
|
||||
console.log(fmtRow("playbook_memory.entries", before.pm_entries, after.pm_entries));
|
||||
console.log(fmtRow("playbook_memory.names", before.pm_names, after.pm_names));
|
||||
console.log(fmtRow("successful_playbooks.rows", before.sp_rows, after.sp_rows));
|
||||
console.log(fmtRow("tools/audit.count", before.audit_count, after.audit_count));
|
||||
console.log(fmtRow("access/audit.count", before.access_count, after.access_count));
|
||||
console.log(fmtRow("journal.events", before.journal_count, after.journal_count));
|
||||
console.log(fmtRow("storage/errors.count", before.storage_errors,after.storage_errors));
|
||||
|
||||
// ── L5: playbook_memory grew? ──
|
||||
if (d.pm_entries === 1) note("L5 playbook_memory growth", true, "+1 entry as expected");
|
||||
else note("L5 playbook_memory growth", d.pm_entries > 0,
|
||||
`delta=${d.pm_entries} (expected 1 — seed-after-log path)`);
|
||||
|
||||
// ── L4: successful_playbooks SQL row appeared? ──
|
||||
if (d.sp_rows >= 1) note("L4 successful_playbooks SQL", true, `+${d.sp_rows} row(s)`);
|
||||
else note("L4 successful_playbooks SQL", false,
|
||||
`delta=${d.sp_rows} — Bun /log claims success but SQL table didn't grow. Recruiter querying via SQL would miss this fill.`);
|
||||
|
||||
// ── L9: storage errors stayed quiet ──
|
||||
if (d.storage_errors === 0) note("L9 storage error journal", true, "no new bucket op errors");
|
||||
else note("L9 storage error journal", false, `+${d.storage_errors} new errors`);
|
||||
|
||||
// ── L10: Phase 17 profile activation ──
|
||||
console.log(`\n▶ L10 — Activate profile ${PROFILE_ID}`);
|
||||
const act = await postJSON<any>(`${GATEWAY}/vectors/profile/${PROFILE_ID}/activate`, {});
|
||||
if (act?._error) note("L10 profile activation", false, `error: ${act._error}`);
|
||||
else note("L10 profile activation", true,
|
||||
`warmed=${(act?.warmed_indexes ?? []).length} duration_ms=${act?.duration_ms ?? "?"}`);
|
||||
|
||||
// ── L11: Bun /search again — boost should now lift PICKED_WORKER ──
|
||||
console.log(`\n▶ L11 — Bun /search second time (boost lift verification)`);
|
||||
const search2 = await postJSON<any>(`${BUN}/search`, {
|
||||
question: `Welder in ${OP_CITY}, ${OP_STATE}`,
|
||||
sql_filter, top_k: 10, generate: false,
|
||||
id_column: "worker_id", dataset: "workers_500k", use_playbook_memory: true,
|
||||
});
|
||||
const sources2 = search2?.sources ?? [];
|
||||
const pickedHit = sources2.find((s: any) => String(s.chunk_text ?? "").includes(PICKED_WORKER));
|
||||
if (!pickedHit) {
|
||||
note("L11 boost lifts logged pick (Bun)", false,
|
||||
`${PICKED_WORKER} not in top-10 via Bun /search. Could be Bun-not-forwarding-playbook_memory_k bug from L1.`);
|
||||
} else if ((pickedHit.playbook_boost ?? 0) > 0) {
|
||||
note("L11 boost lifts logged pick (Bun)", true,
|
||||
`${PICKED_WORKER} boost=+${(pickedHit.playbook_boost as number).toFixed(3)} cites=${(pickedHit.playbook_citations ?? []).length}`);
|
||||
} else {
|
||||
note("L11 boost lifts logged pick (Bun)", false,
|
||||
`${PICKED_WORKER} present but boost=0 — playbook_memory_k forward bug likely`);
|
||||
}
|
||||
|
||||
// Same probe via direct gateway to isolate Bun vs gateway
|
||||
const direct2 = await postJSON<any>(`${GATEWAY}/vectors/hybrid`, {
|
||||
index_name: "workers_500k_v1", filter_dataset: "workers_500k", id_column: "worker_id",
|
||||
sql_filter, question: `Welder in ${OP_CITY}, ${OP_STATE}`,
|
||||
top_k: 10, generate: false, use_playbook_memory: true, playbook_memory_k: 15,
|
||||
});
|
||||
const sources2d = direct2?.sources ?? [];
|
||||
const pickedHitD = sources2d.find((s: any) => String(s.chunk_text ?? "").includes(PICKED_WORKER));
|
||||
if (pickedHitD && (pickedHitD.playbook_boost ?? 0) > 0) {
|
||||
note("L11b boost via direct gateway", true,
|
||||
`${PICKED_WORKER} boost=+${(pickedHitD.playbook_boost as number).toFixed(3)} cites=${(pickedHitD.playbook_citations ?? []).length}`);
|
||||
} else {
|
||||
note("L11b boost via direct gateway", false, `direct call also did not boost ${PICKED_WORKER}`);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─────────────────────── verifier (fresh agent) ───────────────────────
|
||||
|
||||
async function verifierJudgment(trace: TraceResult[]): Promise<{ verdict: string; confidence: number }> {
|
||||
const summary = trace.map(t => ` ${t.ok ? "ok" : "FAIL"} ${t.layer}: ${t.detail}`).join("\n");
|
||||
const prompt = `You are the CHAIN-OF-CUSTODY VERIFIER agent. A real recruiter operation was just
|
||||
traced through every layer of the staffing substrate. Read the per-layer results and judge
|
||||
whether the system kept chain of custody intact (every layer recorded the operation as
|
||||
expected) or where it broke.
|
||||
|
||||
Per-layer trace:
|
||||
${summary}
|
||||
|
||||
Reply with ONE JSON object only:
|
||||
{"verdict": "<one tight sentence — what's the integrity status>", "confidence": 0-100}
|
||||
|
||||
Be specific about which layer broke if any. confidence is how sure you are about the verdict.`;
|
||||
|
||||
try {
|
||||
const raw = await generate("qwen2.5:latest", prompt, { temperature: 0.1, max_tokens: 200 });
|
||||
const start = raw.indexOf("{"), end = raw.lastIndexOf("}");
|
||||
if (start < 0 || end <= start) return { verdict: "verifier could not produce JSON", confidence: 0 };
|
||||
const j = JSON.parse(raw.slice(start, end + 1));
|
||||
return { verdict: j.verdict ?? "no verdict", confidence: Number(j.confidence) || 0 };
|
||||
} catch (e) {
|
||||
return { verdict: `verifier error: ${(e as Error).message}`, confidence: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────── main ───────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log(`▶ Chain-of-custody trace — single real recruiter operation through every layer`);
|
||||
|
||||
const trace = await runTrace();
|
||||
|
||||
console.log(`\n▶ L12 — Verifier (fresh qwen2.5 agent reads the cross-layer trace)`);
|
||||
const v = await verifierJudgment(trace);
|
||||
console.log(` verdict (${v.confidence}%): ${v.verdict}`);
|
||||
|
||||
// Hard gate: any explicit CHAIN BREAK note = fail
|
||||
const breaks = trace.filter(t => !t.ok && t.layer.startsWith("CHAIN BREAK"));
|
||||
const fails = trace.filter(t => !t.ok);
|
||||
|
||||
console.log(`\n▶ Summary:`);
|
||||
console.log(` passing layers: ${trace.filter(t => t.ok).length}/${trace.length}`);
|
||||
console.log(` chain breaks: ${breaks.length}`);
|
||||
console.log(` total failures: ${fails.length}`);
|
||||
console.log(` verifier confidence: ${v.confidence}%`);
|
||||
|
||||
if (breaks.length > 0) {
|
||||
console.log(`\n✗ Chain of custody BROKEN at ${breaks.length} layer(s):`);
|
||||
for (const b of breaks) console.log(` - ${b.layer}: ${b.detail}`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (fails.length > 0) {
|
||||
console.log(`\n◑ Trace completed with ${fails.length} non-blocking failures (no formal chain break)`);
|
||||
process.exit(0);
|
||||
}
|
||||
console.log(`\n✓ Chain of custody intact across all layers`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error(`\n✗ ${(e as Error).message}`);
|
||||
if ((e as any).stack) console.error((e as any).stack);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,469 @@
|
||||
// Network proving: continuous build → verify → repeat with hot-swap profile.
|
||||
//
|
||||
// J's framing: "have them guide each other, when the test is complete we have
|
||||
// a successful playbook, then spin up another agent that tests the viability
|
||||
// of our network with the playbook and the hot-swap profile. Keep spinning up
|
||||
// agents and testing — pass theory, real-world execution, not isolated unit
|
||||
// tests."
|
||||
//
|
||||
// Each round = TWO phases:
|
||||
//
|
||||
// 1. BUILD phase. Two agents (mistral executor + qwen2.5 reviewer) work
|
||||
// on a real staffing fill task. They guide each other via the critique
|
||||
// loop. On consensus → seal a playbook with CANONICAL short seed text
|
||||
// (the Pass 1 lesson — verbose seeds silently kill boost). Real Ollama,
|
||||
// real workers_500k, real /vectors/hybrid path.
|
||||
//
|
||||
// 2. VERIFY phase. A FRESH qwen2.5 agent spins up, activates the
|
||||
// staffing-recruiter profile (Phase 17 hot-swap), runs a probe query
|
||||
// against the same network, and judges from the live response whether
|
||||
// prior rounds' playbooks actually surface relevant workers higher.
|
||||
// The verifier writes a verdict: did the network learn?
|
||||
//
|
||||
// Three rounds, progressively harder:
|
||||
// R0: Welder x2 in Toledo, OH — baseline
|
||||
// R1: Welder x2 in Cleveland, OH — same role, different city
|
||||
// → tests geo discrimination
|
||||
// (Toledo workers MUST NOT
|
||||
// bleed into Cleveland boost)
|
||||
// R2: Welder x3 in Toledo, OH — re-fill same city, bigger
|
||||
// count → tests compounding
|
||||
// (R0's endorsements should
|
||||
// still rank up here)
|
||||
//
|
||||
// Run: bun run tests/multi-agent/network_proving.ts
|
||||
//
|
||||
// Fail-fast: any HTTP error or model crash bubbles to top-level, exits 1.
|
||||
|
||||
import {
|
||||
type LogEntry,
|
||||
type TaskSpec,
|
||||
type Action,
|
||||
type Fill,
|
||||
GATEWAY,
|
||||
generate,
|
||||
parseAction,
|
||||
executorPrompt,
|
||||
reviewerPrompt,
|
||||
sqlQuery,
|
||||
callTool,
|
||||
} from "./agent.ts";
|
||||
|
||||
const EXECUTOR_MODEL = "mistral:latest";
|
||||
const REVIEWER_MODEL = "qwen2.5:latest";
|
||||
const VERIFIER_MODEL = "qwen2.5:latest";
|
||||
const PROFILE_ID = "staffing-recruiter";
|
||||
const INDEX_NAME = "workers_500k_v1";
|
||||
const MAX_TURNS = 12;
|
||||
const MAX_TOOL_ERRORS = 3;
|
||||
const MAX_DRIFTS = 3;
|
||||
|
||||
const TASK_DECK: TaskSpec[] = [
|
||||
{
|
||||
id: "R0", operation: "fill: Welder x2 in Toledo, OH",
|
||||
target_role: "Welder", target_count: 2, target_city: "Toledo", target_state: "OH",
|
||||
approach_hint: "hybrid_search workers_500k_v1 with sql_filter role+state+city, then sql verify",
|
||||
},
|
||||
{
|
||||
id: "R1", operation: "fill: Welder x2 in Cleveland, OH",
|
||||
target_role: "Welder", target_count: 2, target_city: "Cleveland", target_state: "OH",
|
||||
approach_hint: "hybrid_search workers_500k_v1 with sql_filter role+state+city, then sql verify",
|
||||
},
|
||||
{
|
||||
id: "R2", operation: "fill: Welder x3 in Toledo, OH",
|
||||
target_role: "Welder", target_count: 3, target_city: "Toledo", target_state: "OH",
|
||||
approach_hint: "hybrid_search workers_500k_v1 with sql_filter role+state+city, then sql verify",
|
||||
},
|
||||
];
|
||||
|
||||
interface BuildResult {
|
||||
ok: boolean;
|
||||
task: TaskSpec;
|
||||
fills: Fill[];
|
||||
turns: number;
|
||||
duration_secs: number;
|
||||
playbook_id?: string;
|
||||
entries_after_seed?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface VerifyResult {
|
||||
profile_activated: boolean;
|
||||
warmed_indexes: number;
|
||||
probe_boost_total: number; // sum of playbook_boost across top-K
|
||||
probe_boosted_hits: number; // how many hits had boost > 0
|
||||
probe_top_citations: string[]; // playbook_ids cited
|
||||
geo_discrimination_ok: boolean; // when prior playbook is in different city, boost should NOT bleed
|
||||
verdict: string; // qwen2.5's natural-language judgment
|
||||
confidence: number; // 0-100 self-rated
|
||||
duration_secs: number;
|
||||
}
|
||||
|
||||
interface RoundLedger {
|
||||
round: number;
|
||||
task: TaskSpec;
|
||||
build: BuildResult;
|
||||
verify: VerifyResult;
|
||||
score: number; // /10 per round
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
// ─────────────────────── BUILD phase (two-agent loop) ───────────────────────
|
||||
|
||||
async function executeToolCall(name: string, args: Record<string, any>): Promise<any> {
|
||||
if (name === "hybrid_search") {
|
||||
const { sql_filter, question, index_name, k } = args;
|
||||
if (!sql_filter || !question || !index_name) {
|
||||
throw new Error(`hybrid_search needs sql_filter+question+index_name, got keys=${Object.keys(args).join(",")}`);
|
||||
}
|
||||
const r = await fetch(`${GATEWAY}/vectors/hybrid`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
sql_filter, question, index_name,
|
||||
filter_dataset: "workers_500k", id_column: "worker_id",
|
||||
top_k: k ?? 10, generate: false, use_playbook_memory: true,
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`hybrid → ${r.status}: ${await r.text()}`);
|
||||
return r.json();
|
||||
}
|
||||
if (name === "sql") {
|
||||
if (!args.query) throw new Error("sql needs query");
|
||||
if (!/^\s*SELECT/i.test(args.query)) throw new Error("sql allows SELECT only");
|
||||
return sqlQuery(args.query);
|
||||
}
|
||||
return callTool(name, args);
|
||||
}
|
||||
|
||||
function trim(r: any) {
|
||||
if (r && Array.isArray(r.rows)) return { ...r, rows: r.rows.slice(0, 20) };
|
||||
if (r && Array.isArray(r.sources)) return { ...r, sources: r.sources.slice(0, 12) };
|
||||
return r;
|
||||
}
|
||||
|
||||
function fmtTurn(prefix: string, e: Omit<LogEntry, "at">): string {
|
||||
const c: any = e.content ?? {};
|
||||
const head = `[${prefix} t${e.turn.toString().padStart(2, "0")} ${e.role.padEnd(8)} ${e.kind.padEnd(14)}]`;
|
||||
if (e.kind === "tool_call") return `${head} ${c.tool}(${JSON.stringify(c.args ?? {}).slice(0, 70)})`;
|
||||
if (e.kind === "tool_result") {
|
||||
if (c.error) return `${head} error: ${c.error}`;
|
||||
if (Array.isArray(c.sources)) return `${head} hybrid sql=${c.sql_matches} reranked=${c.vector_reranked}`;
|
||||
if (Array.isArray(c.rows)) return `${head} sql ${c.rows.length} rows`;
|
||||
return `${head} ${JSON.stringify(c).slice(0, 70)}`;
|
||||
}
|
||||
if (e.kind === "critique") return `${head} verdict=${c.verdict} ${(c.notes ?? "").slice(0, 50)}`;
|
||||
if (e.kind === "propose_done") return `${head} ${(c.fills ?? []).length} fills: ${(c.fills ?? []).map((f: Fill) => f.name).join(", ")}`;
|
||||
if (e.kind === "consensus_done") return `${head} ✓`;
|
||||
if (e.kind === "plan") return `${head} ${(c.steps ?? []).length} steps`;
|
||||
return `${head} ${JSON.stringify(c).slice(0, 60)}`;
|
||||
}
|
||||
|
||||
async function buildPhase(task: TaskSpec, prefix: string): Promise<BuildResult> {
|
||||
const t0 = Date.now();
|
||||
const log: LogEntry[] = [];
|
||||
let turn = 0, sealed: { fills: Fill[]; approach: string } | null = null;
|
||||
let toolErrors = 0, drifts = 0;
|
||||
|
||||
const append = (e: Omit<LogEntry, "at">): LogEntry => {
|
||||
const full: LogEntry = { ...e, at: new Date().toISOString() };
|
||||
log.push(full);
|
||||
console.log(fmtTurn(prefix, e));
|
||||
return full;
|
||||
};
|
||||
|
||||
try {
|
||||
while (turn < MAX_TURNS && !sealed) {
|
||||
turn += 1;
|
||||
|
||||
const execRaw = await generate(EXECUTOR_MODEL, executorPrompt(task, log), { temperature: 0.2, max_tokens: 600 });
|
||||
const execAction = parseAction(execRaw, "executor");
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: execAction.kind as any, content: execAction });
|
||||
|
||||
if (execAction.kind === "tool_call") {
|
||||
try {
|
||||
const r = await executeToolCall(execAction.tool, execAction.args);
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "tool_result", content: trim(r) });
|
||||
toolErrors = 0;
|
||||
} catch (e) {
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "tool_result",
|
||||
content: { error: (e as Error).message, tool: execAction.tool, args: execAction.args } });
|
||||
toolErrors += 1;
|
||||
if (toolErrors >= MAX_TOOL_ERRORS) throw new Error(`${MAX_TOOL_ERRORS} consecutive tool errors`);
|
||||
}
|
||||
}
|
||||
|
||||
const revRaw = await generate(REVIEWER_MODEL, reviewerPrompt(task, log), { temperature: 0.1, max_tokens: 400 });
|
||||
const revAction = parseAction(revRaw, "reviewer");
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "critique", content: revAction });
|
||||
|
||||
if (revAction.kind !== "critique") throw new Error(`reviewer non-critique`);
|
||||
if (revAction.verdict === "drift") {
|
||||
drifts += 1;
|
||||
if (drifts >= MAX_DRIFTS) throw new Error(`${MAX_DRIFTS} consecutive drifts`);
|
||||
} else drifts = 0;
|
||||
|
||||
if (execAction.kind === "propose_done" && revAction.verdict === "approve_done") {
|
||||
if (execAction.fills.length !== task.target_count) {
|
||||
throw new Error(`fills=${execAction.fills.length} target=${task.target_count}`);
|
||||
}
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "consensus_done", content: { fills: execAction.fills } });
|
||||
sealed = { fills: execAction.fills, approach: (execAction as any).rationale ?? "multi-agent" };
|
||||
}
|
||||
}
|
||||
|
||||
if (!sealed) throw new Error(`no consensus after ${MAX_TURNS} turns`);
|
||||
|
||||
// Phase 19 seed — CANONICAL short text (Pass 1 lesson). The verbose
|
||||
// executor rationale stays out of the embedding; we keep a separate
|
||||
// human-readable record in the playbook log.
|
||||
const canonicalApproach = `${task.target_role.toLowerCase()} fill via hybrid search`;
|
||||
const canonicalContext = `${task.target_role} fill in ${task.target_city}, ${task.target_state}`;
|
||||
let playbook_id: string | undefined;
|
||||
let entries_after_seed: number | undefined;
|
||||
try {
|
||||
const sr = await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: task.operation,
|
||||
approach: canonicalApproach,
|
||||
context: canonicalContext,
|
||||
endorsed_names: sealed.fills.map(f => f.name),
|
||||
append: true,
|
||||
}),
|
||||
});
|
||||
if (sr.ok) {
|
||||
const j = await sr.json() as any;
|
||||
playbook_id = j.playbook_id;
|
||||
entries_after_seed = j.entries_after;
|
||||
console.log(`[${prefix}] ↳ seeded id=${playbook_id} entries=${entries_after_seed}`);
|
||||
} else {
|
||||
console.warn(`[${prefix}] ↳ seed failed: ${sr.status} ${await sr.text()}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[${prefix}] ↳ seed errored: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true, task, fills: sealed.fills, turns: turn,
|
||||
duration_secs: Math.round((Date.now() - t0) / 1000),
|
||||
playbook_id, entries_after_seed,
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false, task, fills: [], turns: turn,
|
||||
duration_secs: Math.round((Date.now() - t0) / 1000),
|
||||
error: (e as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────── VERIFY phase (fresh single agent) ───────────────────────
|
||||
|
||||
async function activateProfile(): Promise<{ ok: boolean; warmed: number; ms: number }> {
|
||||
const t0 = Date.now();
|
||||
const r = await fetch(`${GATEWAY}/vectors/profile/${PROFILE_ID}/activate`, { method: "POST" });
|
||||
const ms = Date.now() - t0;
|
||||
if (!r.ok) {
|
||||
console.warn(`profile activation failed: ${r.status} ${await r.text()}`);
|
||||
return { ok: false, warmed: 0, ms };
|
||||
}
|
||||
const j = await r.json() as any;
|
||||
return { ok: true, warmed: (j.warmed_indexes ?? []).length, ms };
|
||||
}
|
||||
|
||||
async function probeWithBoost(task: TaskSpec) {
|
||||
const sql_filter = `role = '${task.target_role.replace(/'/g, "''")}' `
|
||||
+ `AND state = '${task.target_state}' `
|
||||
+ `AND city = '${task.target_city.replace(/'/g, "''")}'`;
|
||||
const r = await fetch(`${GATEWAY}/vectors/hybrid`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
index_name: INDEX_NAME, filter_dataset: "workers_500k", id_column: "worker_id",
|
||||
sql_filter, question: `${task.target_role} in ${task.target_city}, ${task.target_state}`,
|
||||
top_k: 10, generate: false, use_playbook_memory: true, playbook_memory_k: 15,
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`probe → ${r.status}: ${await r.text()}`);
|
||||
const j = (await r.json()) as any;
|
||||
const sources: any[] = j.sources ?? [];
|
||||
const boostedHits = sources.filter(s => (s.playbook_boost ?? 0) > 0).length;
|
||||
const totalBoost = sources.reduce((s, x) => s + (x.playbook_boost ?? 0), 0);
|
||||
const cites = Array.from(new Set(sources.flatMap(s => s.playbook_citations ?? []))).slice(0, 5);
|
||||
const topNames = sources.slice(0, 5).map(s => {
|
||||
const t = String(s.chunk_text ?? "");
|
||||
return t.split("—")[0]?.trim() ?? s.doc_id;
|
||||
});
|
||||
return { sources, boostedHits, totalBoost, cites, topNames };
|
||||
}
|
||||
|
||||
// Verifier prompt — fresh agent, no shared log with the build pair. It
|
||||
// gets the round's task, the prior rounds' sealed playbooks, and the live
|
||||
// probe result, and renders a human-readable verdict with a confidence.
|
||||
function verifierPrompt(task: TaskSpec, priorPlaybooks: Array<{op: string; fills: string[]}>,
|
||||
probe: { boostedHits: number; totalBoost: number; cites: string[]; topNames: string[] }
|
||||
): string {
|
||||
const priorBlock = priorPlaybooks.length === 0
|
||||
? "(no prior playbooks — this is the first round)"
|
||||
: priorPlaybooks.map((p, i) => ` ${i+1}. ${p.op} → endorsed [${p.fills.join(", ")}]`).join("\n");
|
||||
|
||||
return `You are the VERIFIER agent. A fresh round just sealed a playbook on a real staffing
|
||||
substrate. Your job: judge whether the system learned from prior rounds.
|
||||
|
||||
CURRENT ROUND:
|
||||
task: ${task.operation}
|
||||
in city: ${task.target_city}, ${task.target_state}
|
||||
|
||||
PRIOR PLAYBOOKS (in playbook_memory):
|
||||
${priorBlock}
|
||||
|
||||
I activated the staffing-recruiter profile and ran a hybrid query for this exact task with
|
||||
use_playbook_memory=true. Live result from the substrate:
|
||||
- top-5 surfaced workers: ${probe.topNames.join(", ")}
|
||||
- hits with non-zero playbook_boost: ${probe.boostedHits} / 10
|
||||
- total boost across top-10: ${probe.totalBoost.toFixed(3)}
|
||||
- playbook citations: [${probe.cites.join(", ")}]
|
||||
|
||||
JUDGE:
|
||||
1. If a prior playbook covered this same city + role, the boost should fire on the workers
|
||||
it endorsed (boostedHits > 0, citations non-empty).
|
||||
2. If no prior playbook covers this combo, boost should be ~0 — that means the system is
|
||||
correctly NOT bleeding endorsements across geos.
|
||||
3. Anything in between (e.g. some boost but for the wrong reason) is a partial pass.
|
||||
|
||||
Respond with ONE JSON object only:
|
||||
{"learned": true|false, "verdict": "<one sentence>", "confidence": 0-100}
|
||||
|
||||
learned=true means the network behaved as expected for this round (whether that's "boost fired
|
||||
because it should" or "boost stayed zero because it should"). learned=false means the system
|
||||
either failed to learn from a relevant prior playbook OR bled an irrelevant one. confidence is
|
||||
how sure you are.`;
|
||||
}
|
||||
|
||||
async function verifyPhase(task: TaskSpec, ledger: RoundLedger[]): Promise<VerifyResult> {
|
||||
const t0 = Date.now();
|
||||
const act = await activateProfile();
|
||||
const probe = await probeWithBoost(task);
|
||||
|
||||
// Decide what counts as geo-correct based on prior playbooks
|
||||
const priorMatchesThisGeo = ledger.some(r =>
|
||||
r.build.ok &&
|
||||
r.task.target_city === task.target_city &&
|
||||
r.task.target_state === task.target_state &&
|
||||
r.task.target_role === task.target_role
|
||||
);
|
||||
const priorOtherGeo = ledger.some(r =>
|
||||
r.build.ok &&
|
||||
r.task.target_role === task.target_role &&
|
||||
!(r.task.target_city === task.target_city && r.task.target_state === task.target_state)
|
||||
);
|
||||
|
||||
let geo_discrimination_ok: boolean;
|
||||
if (priorMatchesThisGeo) {
|
||||
geo_discrimination_ok = probe.boostedHits > 0; // expected lift
|
||||
} else if (priorOtherGeo) {
|
||||
geo_discrimination_ok = probe.boostedHits === 0; // must NOT bleed
|
||||
} else {
|
||||
geo_discrimination_ok = true; // no signal expected either way
|
||||
}
|
||||
|
||||
// Spin up the fresh verifier agent
|
||||
const priorPlaybooks = ledger.filter(r => r.build.ok).map(r => ({
|
||||
op: r.task.operation, fills: r.build.fills.map(f => f.name),
|
||||
}));
|
||||
|
||||
let verdict = "verifier failed to respond"; let confidence = 0;
|
||||
try {
|
||||
const raw = await generate(VERIFIER_MODEL, verifierPrompt(task, priorPlaybooks, probe), {
|
||||
temperature: 0.1, max_tokens: 250,
|
||||
});
|
||||
const start = raw.indexOf("{"), end = raw.lastIndexOf("}");
|
||||
if (start >= 0 && end > start) {
|
||||
const j = JSON.parse(raw.slice(start, end + 1));
|
||||
verdict = j.verdict ?? verdict;
|
||||
confidence = Number(j.confidence) || 0;
|
||||
}
|
||||
} catch (e) {
|
||||
verdict = `verifier parse error: ${(e as Error).message}`;
|
||||
}
|
||||
|
||||
return {
|
||||
profile_activated: act.ok,
|
||||
warmed_indexes: act.warmed,
|
||||
probe_boost_total: probe.totalBoost,
|
||||
probe_boosted_hits: probe.boostedHits,
|
||||
probe_top_citations: probe.cites,
|
||||
geo_discrimination_ok,
|
||||
verdict, confidence,
|
||||
duration_secs: Math.round((Date.now() - t0) / 1000),
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────── round scoring ───────────────────────
|
||||
|
||||
function scoreRound(r: RoundLedger): { score: number; notes: string[] } {
|
||||
const notes: string[] = [];
|
||||
let s = 0;
|
||||
if (r.build.ok) { s += 3; notes.push(`✓ build sealed (${r.build.fills.map(f => f.name).join(", ")})`); }
|
||||
else { notes.push(`✗ build failed: ${r.build.error}`); }
|
||||
if (r.build.playbook_id) { s += 1; notes.push(`✓ seeded id=${r.build.playbook_id}`); }
|
||||
if (r.verify.profile_activated) { s += 1; notes.push(`✓ profile activated (warmed=${r.verify.warmed_indexes})`); }
|
||||
if (r.verify.geo_discrimination_ok) { s += 3; notes.push(`✓ geo discrimination correct (boostedHits=${r.verify.probe_boosted_hits})`); }
|
||||
else { notes.push(`✗ geo discrimination failed (boostedHits=${r.verify.probe_boosted_hits})`); }
|
||||
if (r.verify.confidence >= 60) { s += 2; notes.push(`✓ verifier confident (${r.verify.confidence}%): ${r.verify.verdict}`); }
|
||||
else { notes.push(`◑ verifier confidence ${r.verify.confidence}%: ${r.verify.verdict}`); }
|
||||
return { score: s, notes };
|
||||
}
|
||||
|
||||
// ─────────────────────── main loop ───────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log(`▶ Network proving — ${TASK_DECK.length} rounds, profile=${PROFILE_ID}`);
|
||||
console.log(`▶ build pair: ${EXECUTOR_MODEL} + ${REVIEWER_MODEL}; verifier: ${VERIFIER_MODEL}\n`);
|
||||
|
||||
const ledger: RoundLedger[] = [];
|
||||
|
||||
for (let i = 0; i < TASK_DECK.length; i++) {
|
||||
const task = TASK_DECK[i];
|
||||
console.log(`\n══════════ Round ${i} — ${task.operation} ══════════`);
|
||||
|
||||
console.log(`\n[${task.id}] BUILD phase (two agents collaborating)`);
|
||||
const build = await buildPhase(task, task.id);
|
||||
|
||||
console.log(`\n[${task.id}] VERIFY phase (fresh agent + hot-swap profile)`);
|
||||
const verify = await verifyPhase(task, ledger);
|
||||
console.log(` profile=${verify.profile_activated ? "ok" : "fail"} warmed=${verify.warmed_indexes} `
|
||||
+ `boosted=${verify.probe_boosted_hits}/10 totalBoost=${verify.probe_boost_total.toFixed(3)} `
|
||||
+ `cites=${verify.probe_top_citations.length} confidence=${verify.confidence}%`);
|
||||
console.log(` verdict: ${verify.verdict}`);
|
||||
|
||||
const round: RoundLedger = { round: i, task, build, verify, score: 0, notes: [] };
|
||||
const sc = scoreRound(round);
|
||||
round.score = sc.score; round.notes = sc.notes;
|
||||
ledger.push(round);
|
||||
|
||||
console.log(`\n Round ${i} score: ${round.score}/10`);
|
||||
for (const n of round.notes) console.log(` ${n}`);
|
||||
}
|
||||
|
||||
console.log(`\n══════════ Network viability summary ══════════`);
|
||||
const total = ledger.reduce((s, r) => s + r.score, 0);
|
||||
const max = ledger.length * 10;
|
||||
const avg = total / ledger.length;
|
||||
for (const r of ledger) console.log(` R${r.round} ${r.task.target_city.padEnd(10)} ${r.task.target_role.padEnd(20)} ${r.score}/10`);
|
||||
console.log(`\n TOTAL: ${total}/${max} AVG: ${avg.toFixed(1)}/10`);
|
||||
|
||||
// Hard gate: at least 2/3 rounds must show the verifier is confident enough
|
||||
// AND build phase succeeded
|
||||
const passed = ledger.filter(r => r.build.ok && r.score >= 6).length;
|
||||
if (passed < Math.ceil(ledger.length * 2 / 3)) {
|
||||
throw new Error(`network proving gate failed — only ${passed}/${ledger.length} rounds passed (need ≥${Math.ceil(ledger.length * 2 / 3)})`);
|
||||
}
|
||||
console.log(`\n✓ Network proven over ${passed}/${ledger.length} rounds`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error(`\n✗ ${(e as Error).message}`);
|
||||
if ((e as any).stack) console.error((e as any).stack);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,302 @@
|
||||
// Two-agent orchestrator. Both agents run as concurrent async loops
|
||||
// coordinated through a shared in-memory log; one turn of executor then
|
||||
// one turn of reviewer, interleaved until consensus_done, drift-cycle
|
||||
// blown, or hard turn cap. On success writes a playbook JSON; on failure
|
||||
// exits non-zero with the full log for inspection.
|
||||
//
|
||||
// Fail-fast: every caught error is appended to the log AND rethrown, so
|
||||
// the orchestrator top-level catches, dumps, and exits with code 1. The
|
||||
// test harness reads the exit code to decide if the substrate is healthy.
|
||||
|
||||
import {
|
||||
type LogEntry,
|
||||
type TaskSpec,
|
||||
type Action,
|
||||
type Fill,
|
||||
callTool,
|
||||
hybridSearch,
|
||||
sqlQuery,
|
||||
generate,
|
||||
parseAction,
|
||||
executorPrompt,
|
||||
reviewerPrompt,
|
||||
GATEWAY,
|
||||
} from "./agent.ts";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
const EXECUTOR_MODEL = "mistral:latest";
|
||||
const REVIEWER_MODEL = "qwen2.5:latest";
|
||||
const MAX_TURNS = 12; // executor turns; reviewer gets one per
|
||||
const MAX_CONSECUTIVE_DRIFTS = 3; // drift-cycle blown → give up
|
||||
|
||||
// Default task. Override via argv[2] if you want something else; see
|
||||
// `parseTaskFromArg`. Picked from the real-world staffing pattern but
|
||||
// not in the existing successful_playbooks list — this is a fresh fill.
|
||||
// Default task lifted from the production pattern in successful_playbooks.
|
||||
// Toledo, OH has 342 welders in workers_500k so supply is ample — the test
|
||||
// is about collaboration and drift correction, not needle-in-haystack.
|
||||
const DEFAULT_TASK: TaskSpec = {
|
||||
id: `task-${Date.now()}`,
|
||||
operation: "fill: Welder x2 in Toledo, OH",
|
||||
target_role: "Welder",
|
||||
target_count: 2,
|
||||
target_city: "Toledo",
|
||||
target_state: "OH",
|
||||
approach_hint: "hybrid search against workers_500k_v1, narrow by role+city+state+availability, rank semantically",
|
||||
};
|
||||
|
||||
function parseTaskFromArg(): TaskSpec {
|
||||
const arg = process.argv[2];
|
||||
if (!arg) return DEFAULT_TASK;
|
||||
// Accept "role:Welder count:2 city:Columbus state:OH" style for ad-hoc
|
||||
// tasks without standing up a JSON file. Anything more complex, feed
|
||||
// it a JSON path.
|
||||
if (arg.endsWith(".json")) {
|
||||
return JSON.parse(require("node:fs").readFileSync(arg, "utf-8"));
|
||||
}
|
||||
const kv: Record<string, string> = {};
|
||||
for (const token of arg.split(/\s+/)) {
|
||||
const [k, ...v] = token.split(":");
|
||||
kv[k] = v.join(":");
|
||||
}
|
||||
return {
|
||||
id: `task-${Date.now()}`,
|
||||
operation: `fill: ${kv.role} x${kv.count} in ${kv.city}, ${kv.state}`,
|
||||
target_role: kv.role,
|
||||
target_count: Number(kv.count),
|
||||
target_city: kv.city,
|
||||
target_state: kv.state,
|
||||
approach_hint: kv.hint ?? "hybrid search",
|
||||
};
|
||||
}
|
||||
|
||||
// Helper: pretty one-line print for each log entry so the human watching
|
||||
// stdout can follow without pulling the JSONL file.
|
||||
// Defensive one-line formatter. Models sometimes omit optional fields
|
||||
// (rationale, notes), so every access is guarded.
|
||||
function fmt(e: LogEntry): string {
|
||||
const tag = `[t${e.turn.toString().padStart(2, "0")} ${e.role.padEnd(8)} ${e.kind.padEnd(14)}]`;
|
||||
const c = e.content ?? {};
|
||||
const trim = (s: any, n: number) => String(s ?? "").slice(0, n);
|
||||
if (e.kind === "tool_call")
|
||||
return `${tag} ${c.tool}(${JSON.stringify(c.args ?? {}).slice(0, 80)}) — ${trim(c.rationale, 60)}`;
|
||||
if (e.kind === "tool_result") {
|
||||
const rows = c?.rows?.length ?? c?.sources?.length ?? undefined;
|
||||
return `${tag} ${rows !== undefined ? `rows=${rows}` : JSON.stringify(c).slice(0, 80)}`;
|
||||
}
|
||||
if (e.kind === "critique") return `${tag} verdict=${c.verdict} — ${trim(c.notes, 80)}`;
|
||||
if (e.kind === "propose_done")
|
||||
return `${tag} ${c.fills?.length ?? 0} fills: ${(c.fills ?? []).map((f: Fill) => f.name).join(", ")}`;
|
||||
if (e.kind === "consensus_done") return `${tag} ✓`;
|
||||
if (e.kind === "plan") return `${tag} ${c.steps?.length ?? 0} steps: ${(c.steps ?? []).slice(0, 2).join(" / ")}`;
|
||||
if (e.kind === "error") return `${tag} ${c.message ?? c}`;
|
||||
return `${tag} ${JSON.stringify(c).slice(0, 100)}`;
|
||||
}
|
||||
|
||||
// Execute one tool call. The tool catalog in the prompt lists both the
|
||||
// registered Phase 12 tools AND a pseudo-tool "hybrid_search" for the
|
||||
// /vectors/hybrid endpoint — unify here so the executor doesn't need to
|
||||
// know which surface a capability lives on.
|
||||
async function executeToolCall(name: string, args: Record<string, any>): Promise<any> {
|
||||
if (name === "hybrid_search") {
|
||||
const { sql_filter, question, index_name, k } = args;
|
||||
if (!sql_filter || !question || !index_name) {
|
||||
throw new Error(`hybrid_search needs sql_filter + question + index_name, got ${JSON.stringify(args)}`);
|
||||
}
|
||||
// Pass through to /vectors/hybrid. id_column defaults to worker_id
|
||||
// server-side, which is what workers_500k uses.
|
||||
const body: any = { sql_filter, question, index_name, top_k: k ?? 10, generate: false };
|
||||
return (await (await fetch("http://localhost:3100/vectors/hybrid", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
})).json());
|
||||
}
|
||||
if (name === "sql") {
|
||||
const { query } = args;
|
||||
if (!query || typeof query !== "string") throw new Error(`sql needs query (string), got ${JSON.stringify(args)}`);
|
||||
if (!/^\s*SELECT/i.test(query)) throw new Error(`sql tool allows SELECT only: ${query}`);
|
||||
return sqlQuery(query);
|
||||
}
|
||||
// Fall through to Phase 12 registry for any other named tool.
|
||||
return callTool(name, args);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const task = parseTaskFromArg();
|
||||
const log: LogEntry[] = [];
|
||||
let turn = 0;
|
||||
let consecutiveDrifts = 0;
|
||||
let sealed: { fills: Fill[]; approach: string } | null = null;
|
||||
|
||||
const append = (e: Omit<LogEntry, "at">): LogEntry => {
|
||||
const full: LogEntry = { ...e, at: new Date().toISOString() };
|
||||
log.push(full);
|
||||
console.log(fmt(full));
|
||||
return full;
|
||||
};
|
||||
|
||||
console.log(`▶ task: ${task.operation}`);
|
||||
console.log(`▶ executor=${EXECUTOR_MODEL} reviewer=${REVIEWER_MODEL}`);
|
||||
console.log();
|
||||
|
||||
try {
|
||||
while (turn < MAX_TURNS && !sealed) {
|
||||
turn += 1;
|
||||
|
||||
// --- EXECUTOR TURN ---
|
||||
const execRaw = await generate(EXECUTOR_MODEL, executorPrompt(task, log), {
|
||||
temperature: 0.2,
|
||||
max_tokens: 600,
|
||||
});
|
||||
let execAction: Action;
|
||||
try {
|
||||
execAction = parseAction(execRaw, "executor");
|
||||
} catch (e) {
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "error",
|
||||
content: { message: (e as Error).message, raw: execRaw.slice(0, 400) } });
|
||||
throw e;
|
||||
}
|
||||
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL,
|
||||
kind: execAction.kind as any, content: execAction });
|
||||
|
||||
// If tool_call, execute and feed result back into the log. Tool
|
||||
// validation / server errors come back as a tool_result with an
|
||||
// `error` field — the executor reads its own error on the next turn
|
||||
// and self-corrects (e.g. "oh, I forgot the `question` argument").
|
||||
// This is softer than hard-failing the orchestrator: the whole
|
||||
// point of two-agent collaboration is letting agents learn from
|
||||
// immediate feedback instead of crashing the run.
|
||||
if (execAction.kind === "tool_call") {
|
||||
try {
|
||||
const result = await executeToolCall(execAction.tool, execAction.args);
|
||||
const trimmed = trimResult(result);
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL,
|
||||
kind: "tool_result", content: trimmed });
|
||||
} catch (e) {
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL,
|
||||
kind: "tool_result",
|
||||
content: { error: (e as Error).message, tool: execAction.tool, args: execAction.args } });
|
||||
// Count as a soft drift — if the executor keeps throwing tool
|
||||
// errors, consecutiveDrifts still trips the abort.
|
||||
consecutiveDrifts += 1;
|
||||
if (consecutiveDrifts >= MAX_CONSECUTIVE_DRIFTS) {
|
||||
throw new Error(`aborting — ${MAX_CONSECUTIVE_DRIFTS} consecutive tool errors, executor can't self-correct`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- REVIEWER TURN ---
|
||||
const revRaw = await generate(REVIEWER_MODEL, reviewerPrompt(task, log), {
|
||||
temperature: 0.1,
|
||||
max_tokens: 400,
|
||||
});
|
||||
let revAction: Action;
|
||||
try {
|
||||
revAction = parseAction(revRaw, "reviewer");
|
||||
} catch (e) {
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "error",
|
||||
content: { message: (e as Error).message, raw: revRaw.slice(0, 400) } });
|
||||
throw e;
|
||||
}
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL,
|
||||
kind: "critique", content: revAction });
|
||||
|
||||
if (revAction.kind !== "critique") throw new Error(`reviewer emitted non-critique: ${revAction.kind}`);
|
||||
|
||||
if (revAction.verdict === "drift") {
|
||||
consecutiveDrifts += 1;
|
||||
if (consecutiveDrifts >= MAX_CONSECUTIVE_DRIFTS) {
|
||||
throw new Error(`aborting — ${MAX_CONSECUTIVE_DRIFTS} consecutive drift flags, executor can't self-correct`);
|
||||
}
|
||||
} else {
|
||||
consecutiveDrifts = 0;
|
||||
}
|
||||
|
||||
// Consensus: executor proposed done AND reviewer approved.
|
||||
if (execAction.kind === "propose_done" && revAction.verdict === "approve_done") {
|
||||
if (execAction.fills.length !== task.target_count) {
|
||||
throw new Error(`consensus malformed — ${execAction.fills.length} fills vs target ${task.target_count}`);
|
||||
}
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "consensus_done",
|
||||
content: { fills: execAction.fills } });
|
||||
sealed = { fills: execAction.fills, approach: execAction.rationale };
|
||||
}
|
||||
}
|
||||
|
||||
if (!sealed) throw new Error(`no consensus after ${MAX_TURNS} turns — task incomplete`);
|
||||
|
||||
// Write playbook entry matching the successful_playbooks schema.
|
||||
const playbook = {
|
||||
timestamp: new Date().toISOString(),
|
||||
operation: task.operation,
|
||||
approach: sealed.approach,
|
||||
result: `${sealed.fills.length}/${task.target_count} filled → ${sealed.fills.map(f => f.name).join(", ")}`,
|
||||
context: `executor=${EXECUTOR_MODEL} reviewer=${REVIEWER_MODEL} turns=${turn}`,
|
||||
task,
|
||||
fills: sealed.fills,
|
||||
log,
|
||||
};
|
||||
await mkdir("./tests/multi-agent/playbooks", { recursive: true });
|
||||
const path = join("./tests/multi-agent/playbooks", `${task.id}.json`);
|
||||
await writeFile(path, JSON.stringify(playbook, null, 2));
|
||||
console.log(`\n✓ playbook written: ${path}`);
|
||||
console.log(` ${playbook.result}`);
|
||||
|
||||
// Phase 19.5: write-through to playbook_memory. The sealed fills are
|
||||
// the endorsement; next semantically-similar query will surface them
|
||||
// higher. /seed bypasses the successful_playbooks ingest round-trip
|
||||
// — when that ingest path ships, this block should switch to append
|
||||
// + rebuild instead.
|
||||
try {
|
||||
// Seed context is what the embedding model actually sees alongside
|
||||
// the operation — so it has to carry task-semantic content (role,
|
||||
// city, scenario) rather than orchestrator bookkeeping. We stash
|
||||
// the bookkeeping in the full playbook JSON instead (see playbook
|
||||
// object above) where operators can grep it without it polluting
|
||||
// the ranking signal.
|
||||
const seedContext = task.approach_hint
|
||||
?? `${task.target_role} fill in ${task.target_city}, ${task.target_state}`;
|
||||
const seedRes = await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: task.operation,
|
||||
approach: sealed.approach || "multi-agent → hybrid search",
|
||||
context: seedContext,
|
||||
endorsed_names: sealed.fills.map(f => f.name),
|
||||
append: true,
|
||||
}),
|
||||
});
|
||||
if (seedRes.ok) {
|
||||
const j = await seedRes.json() as any;
|
||||
console.log(` ↳ playbook_memory seeded: id=${j.playbook_id} entries=${j.entries_after}`);
|
||||
} else {
|
||||
console.warn(` ↳ playbook_memory seed failed: ${seedRes.status} ${await seedRes.text()}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(` ↳ playbook_memory seed errored: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
} catch (e) {
|
||||
console.error(`\n✗ ${(e as Error).message}`);
|
||||
// Still persist the log for inspection.
|
||||
await mkdir("./tests/multi-agent/playbooks", { recursive: true });
|
||||
const path = join("./tests/multi-agent/playbooks", `${task.id}-FAILED.json`);
|
||||
await writeFile(path, JSON.stringify({ task, error: (e as Error).message, log }, null, 2));
|
||||
console.error(` log dumped: ${path}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function trimResult(r: any): any {
|
||||
if (r && Array.isArray(r.rows)) {
|
||||
return { ...r, rows: r.rows.slice(0, 20), _trimmed: r.rows.length > 20 ? `${r.rows.length - 20} more rows` : undefined };
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,400 @@
|
||||
// Two-agent x two-tasks parallel real-world test with per-playbook rating.
|
||||
//
|
||||
// Spawns two independent (executor, reviewer) pairs concurrently, each
|
||||
// driving a different staffing fill against the live substrate. After
|
||||
// each pair seals a playbook, verifies the fill against workers_500k,
|
||||
// confirms the seed reached playbook_memory, and re-runs the same query
|
||||
// with use_playbook_memory=true to prove the boost fires.
|
||||
//
|
||||
// Errors fail fast — any HTTP error, parse error, or rating failure is
|
||||
// rethrown so bun exits non-zero. Run with:
|
||||
//
|
||||
// bun run tests/multi-agent/run_e2e_rated.ts
|
||||
//
|
||||
// VRAM note: both pairs call the same two Ollama models (mistral +
|
||||
// qwen2.5). Ollama queues at the model level, so "parallel" is concurrent
|
||||
// orchestration, not concurrent inference — the loops interleave on the
|
||||
// shared models. That's intentional: it stresses the same realistic
|
||||
// path two staffing coordinators would hit if they both opened the app
|
||||
// at 8am.
|
||||
|
||||
import {
|
||||
type LogEntry,
|
||||
type TaskSpec,
|
||||
type Action,
|
||||
type Fill,
|
||||
GATEWAY,
|
||||
generate,
|
||||
parseAction,
|
||||
executorPrompt,
|
||||
reviewerPrompt,
|
||||
sqlQuery,
|
||||
callTool,
|
||||
} from "./agent.ts";
|
||||
|
||||
const EXECUTOR_MODEL = "mistral:latest";
|
||||
const REVIEWER_MODEL = "qwen2.5:latest";
|
||||
const MAX_TURNS = 12;
|
||||
const MAX_CONSECUTIVE_DRIFTS = 3;
|
||||
const INDEX_NAME = "workers_500k_v1";
|
||||
|
||||
interface RunResult {
|
||||
task: TaskSpec;
|
||||
ok: boolean;
|
||||
turns: number;
|
||||
duration_secs: number;
|
||||
fills: Fill[];
|
||||
log: LogEntry[];
|
||||
approach: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ────────────────────────── orchestrator (function form) ──────────────────────────
|
||||
|
||||
async function runOrchestrator(task: TaskSpec, prefix: string): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
const log: LogEntry[] = [];
|
||||
let turn = 0;
|
||||
let consecutiveDrifts = 0;
|
||||
// Track tool errors separately from drift verdicts. Reviewer saying
|
||||
// "continue" or "approve_done" should NOT reset a streak of malformed
|
||||
// tool calls — that's a different failure mode (model can't form the
|
||||
// call) than "executor is on the wrong path" (model is off-topic).
|
||||
let consecutiveToolErrors = 0;
|
||||
let sealed: { fills: Fill[]; approach: string } | null = null;
|
||||
|
||||
const append = (e: Omit<LogEntry, "at">): LogEntry => {
|
||||
const full: LogEntry = { ...e, at: new Date().toISOString() };
|
||||
log.push(full);
|
||||
console.log(`[${prefix}] [t${e.turn.toString().padStart(2, "0")} ${e.role.padEnd(8)} ${e.kind.padEnd(14)}] ${shortContent(e)}`);
|
||||
return full;
|
||||
};
|
||||
|
||||
try {
|
||||
while (turn < MAX_TURNS && !sealed) {
|
||||
turn += 1;
|
||||
|
||||
// Executor
|
||||
const execRaw = await generate(EXECUTOR_MODEL, executorPrompt(task, log), { temperature: 0.2, max_tokens: 600 });
|
||||
const execAction = parseAction(execRaw, "executor");
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: execAction.kind as any, content: execAction });
|
||||
|
||||
if (execAction.kind === "tool_call") {
|
||||
try {
|
||||
const result = await executeToolCall(execAction.tool, execAction.args);
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "tool_result", content: trimResult(result) });
|
||||
consecutiveToolErrors = 0;
|
||||
} catch (e) {
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "tool_result",
|
||||
content: { error: (e as Error).message, tool: execAction.tool, args: execAction.args } });
|
||||
consecutiveToolErrors += 1;
|
||||
if (consecutiveToolErrors >= MAX_CONSECUTIVE_DRIFTS) {
|
||||
throw new Error(`${MAX_CONSECUTIVE_DRIFTS} consecutive tool errors — executor can't form a valid call`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reviewer
|
||||
const revRaw = await generate(REVIEWER_MODEL, reviewerPrompt(task, log), { temperature: 0.1, max_tokens: 400 });
|
||||
const revAction = parseAction(revRaw, "reviewer");
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "critique", content: revAction });
|
||||
|
||||
if (revAction.kind !== "critique") throw new Error(`reviewer non-critique: ${revAction.kind}`);
|
||||
if (revAction.verdict === "drift") {
|
||||
consecutiveDrifts += 1;
|
||||
if (consecutiveDrifts >= MAX_CONSECUTIVE_DRIFTS) throw new Error(`${MAX_CONSECUTIVE_DRIFTS} consecutive drifts`);
|
||||
} else consecutiveDrifts = 0;
|
||||
|
||||
if (execAction.kind === "propose_done" && revAction.verdict === "approve_done") {
|
||||
if (execAction.fills.length !== task.target_count) {
|
||||
throw new Error(`fills=${execAction.fills.length} target=${task.target_count}`);
|
||||
}
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "consensus_done", content: { fills: execAction.fills } });
|
||||
sealed = { fills: execAction.fills, approach: (execAction as any).rationale ?? "multi-agent → hybrid" };
|
||||
}
|
||||
}
|
||||
|
||||
if (!sealed) throw new Error(`no consensus after ${MAX_TURNS} turns`);
|
||||
|
||||
// Phase 19 write-through: seed playbook_memory so the next semantically
|
||||
// similar query benefits from this fill. Mirrors orchestrator.ts. Names
|
||||
// are the consensus fills' display names — that's what the boost-key
|
||||
// matcher (city, state, name) will look up against worker chunks.
|
||||
try {
|
||||
const seedRes = await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: task.operation,
|
||||
approach: sealed.approach || "multi-agent → hybrid search",
|
||||
context: task.approach_hint ?? `${task.target_role} fill in ${task.target_city}, ${task.target_state}`,
|
||||
endorsed_names: sealed.fills.map(f => f.name),
|
||||
append: true,
|
||||
}),
|
||||
});
|
||||
if (!seedRes.ok) {
|
||||
console.warn(`[${prefix}] seed warning: ${seedRes.status} ${await seedRes.text()}`);
|
||||
} else {
|
||||
const j = await seedRes.json() as any;
|
||||
console.log(`[${prefix}] ↳ seeded playbook_memory: id=${j.playbook_id} entries=${j.entries_after}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[${prefix}] seed errored: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
task, ok: true, turns: turn, fills: sealed.fills, approach: sealed.approach,
|
||||
duration_secs: Math.round((Date.now() - start) / 1000), log,
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
task, ok: false, turns: turn, fills: [], approach: "",
|
||||
duration_secs: Math.round((Date.now() - start) / 1000), log,
|
||||
error: (e as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function executeToolCall(name: string, args: Record<string, any>): Promise<any> {
|
||||
if (name === "hybrid_search") {
|
||||
const { sql_filter, question, index_name, k } = args;
|
||||
if (!sql_filter || !question || !index_name) throw new Error(`hybrid_search needs sql_filter+question+index_name`);
|
||||
const r = await fetch(`${GATEWAY}/vectors/hybrid`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ sql_filter, question, index_name, top_k: k ?? 10, generate: false, use_playbook_memory: true }),
|
||||
});
|
||||
if (!r.ok) throw new Error(`hybrid_search → ${r.status}: ${await r.text()}`);
|
||||
return r.json();
|
||||
}
|
||||
if (name === "sql") {
|
||||
if (!args.query || typeof args.query !== "string") throw new Error("sql needs query");
|
||||
if (!/^\s*SELECT/i.test(args.query)) throw new Error("sql allows SELECT only");
|
||||
return sqlQuery(args.query);
|
||||
}
|
||||
return callTool(name, args);
|
||||
}
|
||||
|
||||
function trimResult(r: any): any {
|
||||
if (r && Array.isArray(r.rows)) return { ...r, rows: r.rows.slice(0, 20) };
|
||||
if (r && Array.isArray(r.sources)) return { ...r, sources: r.sources.slice(0, 12) };
|
||||
return r;
|
||||
}
|
||||
|
||||
function shortContent(e: Omit<LogEntry, "at">): string {
|
||||
const c: any = e.content ?? {};
|
||||
if (e.kind === "tool_call") return `${c.tool}(${JSON.stringify(c.args ?? {}).slice(0, 70)})`;
|
||||
if (e.kind === "tool_result") {
|
||||
if (c.error) return `error: ${c.error}`;
|
||||
if (Array.isArray(c.sources)) return `hybrid sql=${c.sql_matches} reranked=${c.vector_reranked}`;
|
||||
if (Array.isArray(c.rows)) return `sql ${c.rows.length} rows`;
|
||||
return JSON.stringify(c).slice(0, 80);
|
||||
}
|
||||
if (e.kind === "critique") return `verdict=${c.verdict} ${(c.notes ?? "").slice(0, 60)}`;
|
||||
if (e.kind === "propose_done") return `${(c.fills ?? []).length} fills: ${(c.fills ?? []).map((f: Fill) => f.name).join(", ")}`;
|
||||
if (e.kind === "consensus_done") return "✓";
|
||||
if (e.kind === "plan") return `${(c.steps ?? []).length} steps`;
|
||||
return JSON.stringify(c).slice(0, 80);
|
||||
}
|
||||
|
||||
// ────────────────────────── playbook rating ──────────────────────────
|
||||
|
||||
interface Rating {
|
||||
geo: number; // 0-2: fills actually in target city/state
|
||||
authenticity: number; // 0-2: fills' worker_ids exist in workers_500k
|
||||
persistence: number; // 0-2: playbook_memory entry count grew correctly
|
||||
boost_firing: number; // 0-3: follow-up query shows non-zero boost
|
||||
speed: number; // 0-1: completed under 4 min
|
||||
total: number; // /10
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
interface MemoryStats { entries: number; total_names_endorsed: number }
|
||||
|
||||
async function fetchMemoryStats(): Promise<MemoryStats> {
|
||||
const r = await fetch(`${GATEWAY}/vectors/playbook_memory/stats`);
|
||||
if (!r.ok) throw new Error(`stats → ${r.status}`);
|
||||
return r.json() as Promise<MemoryStats>;
|
||||
}
|
||||
|
||||
// Try to resolve a fill's candidate_id to a workers_500k row. Accepts
|
||||
// "W500K-7995" (vector doc_id with prefix) and "7995" (raw worker_id).
|
||||
async function lookupWorker(candidate_id: string): Promise<{ worker_id: number; name: string; city: string; state: string; role: string } | null> {
|
||||
const numStr = candidate_id.replace(/^W500K-/i, "").replace(/[^\d]/g, "");
|
||||
if (!numStr) return null;
|
||||
const num = parseInt(numStr, 10);
|
||||
if (!Number.isFinite(num)) return null;
|
||||
const r = await sqlQuery(`SELECT worker_id, name, city, state, role FROM workers_500k WHERE worker_id = ${num} LIMIT 1`);
|
||||
return (r.rows && r.rows[0]) ?? null;
|
||||
}
|
||||
|
||||
// Re-run a hybrid query that mirrors the contract — proves the freshly
|
||||
// seeded playbook actually lifts a future search.
|
||||
async function verifyBoostFires(task: TaskSpec): Promise<{ boostedHits: number; sampleCitations: string[]; topBoost: number }> {
|
||||
// Mirror the contract's actual geo. The playbook stored (city, state)
|
||||
// from the operation; if the verify SQL doesn't restrict to the same
|
||||
// city, the candidate pool may not include the seeded workers and the
|
||||
// boost has nothing to lift. The contract pattern in production also
|
||||
// includes city — recruiters fill specific cities, not whole states.
|
||||
const sql_filter = `role = '${task.target_role.replace(/'/g, "''")}' `
|
||||
+ `AND state = '${task.target_state}' `
|
||||
+ `AND city = '${task.target_city.replace(/'/g, "''")}'`;
|
||||
const r = await fetch(`${GATEWAY}/vectors/hybrid`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
index_name: INDEX_NAME, filter_dataset: "workers_500k", id_column: "worker_id",
|
||||
sql_filter, question: `${task.target_role} in ${task.target_city}, ${task.target_state}`,
|
||||
top_k: 10, generate: false, use_playbook_memory: true, playbook_memory_k: 15,
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`verify hybrid → ${r.status}: ${await r.text()}`);
|
||||
const j = (await r.json()) as any;
|
||||
const sources: any[] = j.sources ?? [];
|
||||
const boosted = sources.filter(s => (s.playbook_boost ?? 0) > 0);
|
||||
const cites = boosted.flatMap(s => s.playbook_citations ?? []).slice(0, 5);
|
||||
const top = sources.reduce((m, s) => Math.max(m, s.playbook_boost ?? 0), 0);
|
||||
return { boostedHits: boosted.length, sampleCitations: cites, topBoost: top };
|
||||
}
|
||||
|
||||
async function ratePlaybook(
|
||||
result: RunResult,
|
||||
statsBefore: MemoryStats,
|
||||
statsAfter: MemoryStats,
|
||||
): Promise<Rating> {
|
||||
const notes: string[] = [];
|
||||
let geo = 0, authenticity = 0, persistence = 0, boost_firing = 0, speed = 0;
|
||||
|
||||
// 1. Geo + authenticity per fill
|
||||
for (const f of result.fills) {
|
||||
const w = await lookupWorker(f.candidate_id).catch(() => null);
|
||||
if (!w) { notes.push(`✗ candidate_id ${f.candidate_id} not in workers_500k`); continue; }
|
||||
authenticity += 1;
|
||||
if (w.city.toLowerCase() === result.task.target_city.toLowerCase()
|
||||
&& w.state === result.task.target_state) {
|
||||
geo += 1;
|
||||
} else {
|
||||
notes.push(`◑ ${w.name} (id=${w.worker_id}) is in ${w.city}, ${w.state}, not ${result.task.target_city}, ${result.task.target_state}`);
|
||||
}
|
||||
}
|
||||
geo = Math.min(geo, 2);
|
||||
authenticity = Math.min(authenticity, 2);
|
||||
|
||||
// 2. Persistence
|
||||
const grew = statsAfter.entries - statsBefore.entries;
|
||||
if (grew === 1) { persistence = 2; notes.push(`✓ playbook_memory grew by exactly 1`); }
|
||||
else if (grew >= 1) { persistence = 1; notes.push(`◑ playbook_memory grew by ${grew} (expected 1)`); }
|
||||
else { notes.push(`✗ playbook_memory did not grow (before=${statsBefore.entries} after=${statsAfter.entries})`); }
|
||||
|
||||
// 3. Boost firing — re-run the same query and see if it lifts anything
|
||||
const v = await verifyBoostFires(result.task).catch(e => { notes.push(`✗ verify hybrid failed: ${(e as Error).message}`); return null; });
|
||||
if (v) {
|
||||
if (v.boostedHits >= 2) boost_firing = 3;
|
||||
else if (v.boostedHits === 1) boost_firing = 2;
|
||||
else if (v.topBoost > 0) boost_firing = 1;
|
||||
else boost_firing = 0;
|
||||
notes.push(`boost re-query: ${v.boostedHits}/10 hits boosted, top=+${v.topBoost.toFixed(3)}, citations=${v.sampleCitations.slice(0, 3).join(",")}`);
|
||||
}
|
||||
|
||||
// 4. Speed
|
||||
if (result.duration_secs <= 240) speed = 1;
|
||||
else notes.push(`◑ slow: ${result.duration_secs}s (>240)`);
|
||||
|
||||
const total = geo + authenticity + persistence + boost_firing + speed;
|
||||
return { geo, authenticity, persistence, boost_firing, speed, total, notes };
|
||||
}
|
||||
|
||||
function fmtRating(r: Rating): string {
|
||||
return `geo=${r.geo}/2 auth=${r.authenticity}/2 persist=${r.persistence}/2 boost=${r.boost_firing}/3 speed=${r.speed}/1 → ${r.total}/10`;
|
||||
}
|
||||
|
||||
// ────────────────────────── main ──────────────────────────
|
||||
|
||||
async function main() {
|
||||
const taskA: TaskSpec = {
|
||||
id: `e2e-A-${Date.now()}`,
|
||||
operation: "fill: Welder x2 in Toledo, OH",
|
||||
target_role: "Welder", target_count: 2, target_city: "Toledo", target_state: "OH",
|
||||
approach_hint: "hybrid_search against workers_500k_v1 with sql_filter on role+city+state, then sql verify",
|
||||
};
|
||||
const taskB: TaskSpec = {
|
||||
id: `e2e-B-${Date.now()}`,
|
||||
operation: "fill: Forklift Operator x2 in Nashville, TN",
|
||||
target_role: "Forklift Operator", target_count: 2, target_city: "Nashville", target_state: "TN",
|
||||
approach_hint: "hybrid_search against workers_500k_v1 with sql_filter on role+city+state, then sql verify",
|
||||
};
|
||||
|
||||
console.log(`▶ parallel real-world test`);
|
||||
console.log(` A: ${taskA.operation}`);
|
||||
console.log(` B: ${taskB.operation}`);
|
||||
console.log(` models: executor=${EXECUTOR_MODEL} reviewer=${REVIEWER_MODEL}\n`);
|
||||
|
||||
const statsBefore = await fetchMemoryStats();
|
||||
console.log(`▶ playbook_memory before: ${statsBefore.entries} entries, ${statsBefore.total_names_endorsed} endorsed names\n`);
|
||||
|
||||
// Run both pairs in parallel. Each is its own (executor, reviewer)
|
||||
// conversation; they do NOT see each other's logs.
|
||||
const [resA, resB] = await Promise.all([
|
||||
runOrchestrator(taskA, "A"),
|
||||
runOrchestrator(taskB, "B"),
|
||||
]);
|
||||
|
||||
console.log(`\n▶ both orchestrators returned`);
|
||||
console.log(` A: ok=${resA.ok} turns=${resA.turns} ${resA.duration_secs}s ${resA.error ?? ""}`);
|
||||
console.log(` B: ok=${resB.ok} turns=${resB.turns} ${resB.duration_secs}s ${resB.error ?? ""}`);
|
||||
|
||||
if (!resA.ok && !resB.ok) {
|
||||
throw new Error(`both orchestrators failed — substrate or models in bad state`);
|
||||
}
|
||||
|
||||
const statsMid = await fetchMemoryStats();
|
||||
console.log(`\n▶ playbook_memory after both runs: ${statsMid.entries} entries (+${statsMid.entries - statsBefore.entries})\n`);
|
||||
|
||||
// Rate each successful playbook. We compute persistence per task by
|
||||
// splitting the growth — both seeded sequentially-ish, so each should
|
||||
// contribute 1.
|
||||
const ratings: Array<{ id: string; ok: boolean; rating?: Rating; error?: string }> = [];
|
||||
|
||||
if (resA.ok) {
|
||||
const beforeForA: MemoryStats = { entries: statsBefore.entries, total_names_endorsed: statsBefore.total_names_endorsed };
|
||||
const afterForA: MemoryStats = { entries: statsBefore.entries + (resA.fills.length > 0 ? 1 : 0), total_names_endorsed: statsBefore.total_names_endorsed };
|
||||
// Use real measured numbers when they're unambiguous (only one task succeeded)
|
||||
const ra = await ratePlaybook(resA, beforeForA, resB.ok ? afterForA : statsMid);
|
||||
ratings.push({ id: "A", ok: true, rating: ra });
|
||||
} else ratings.push({ id: "A", ok: false, error: resA.error });
|
||||
|
||||
if (resB.ok) {
|
||||
const beforeForB: MemoryStats = resA.ok
|
||||
? { entries: statsBefore.entries + 1, total_names_endorsed: statsBefore.total_names_endorsed }
|
||||
: statsBefore;
|
||||
const rb = await ratePlaybook(resB, beforeForB, statsMid);
|
||||
ratings.push({ id: "B", ok: true, rating: rb });
|
||||
} else ratings.push({ id: "B", ok: false, error: resB.error });
|
||||
|
||||
console.log(`\n▶ Per-playbook ratings:\n`);
|
||||
for (const r of ratings) {
|
||||
if (!r.ok) {
|
||||
console.log(` ${r.id}: FAILED — ${r.error}`);
|
||||
continue;
|
||||
}
|
||||
console.log(` ${r.id}: ${fmtRating(r.rating!)}`);
|
||||
for (const n of r.rating!.notes) console.log(` ${n}`);
|
||||
}
|
||||
|
||||
const totals = ratings.filter(r => r.rating).map(r => r.rating!.total);
|
||||
if (totals.length === 0) {
|
||||
throw new Error(`no playbooks rated — both orchestrators failed`);
|
||||
}
|
||||
const min = Math.min(...totals);
|
||||
const avg = totals.reduce((s, t) => s + t, 0) / totals.length;
|
||||
console.log(`\n▶ Summary: avg=${avg.toFixed(1)}/10 min=${min}/10`);
|
||||
|
||||
// Hard gate: any rating below 5 means the loop is broken end-to-end.
|
||||
if (min < 5) throw new Error(`rating gate failed — min ${min}/10 (need ≥5)`);
|
||||
|
||||
console.log(`\n✓ end-to-end real-world test passed`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error(`\n✗ ${(e as Error).message}`);
|
||||
if ((e as any).stack) console.error((e as any).stack);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,822 @@
|
||||
// A day in the life — the real-world scenario test.
|
||||
//
|
||||
// Runs six events against the live substrate: baseline_fill, recurring,
|
||||
// expansion, emergency, misplacement, retrospective. Each event
|
||||
// exercises a different pressure pattern; each one produces actionable
|
||||
// artifacts (SMS drafts, client emails, dispatch log) alongside the
|
||||
// ranking output; the run as a whole is self-audited at EOD against six
|
||||
// gap categories (supply, embedding, fairness, drift, tool, write-through).
|
||||
//
|
||||
// Design notes:
|
||||
// - Compressed clock. The "08:00" in an event spec is a label for the
|
||||
// output, not a wall-clock gate. The full scenario runs in minutes.
|
||||
// - One script, shared state. Each event mutates the same roster +
|
||||
// gap_signals + artifacts in-memory, then persists at EOD.
|
||||
// - Fail-soft per event. A drift-abort or tool error on one event
|
||||
// records a gap_signal and moves on; we explicitly want to see which
|
||||
// events the substrate can't handle, not abort the whole run.
|
||||
// - Every fill event routes through the same executor/reviewer loop as
|
||||
// the single-task orchestrator — just driven in sequence rather than
|
||||
// standalone, with event-specific extra constraints in the prompt.
|
||||
|
||||
import {
|
||||
type LogEntry,
|
||||
type TaskSpec,
|
||||
type Action,
|
||||
type Fill,
|
||||
callTool,
|
||||
hybridSearch,
|
||||
sqlQuery,
|
||||
generate,
|
||||
parseAction,
|
||||
executorPrompt,
|
||||
reviewerPrompt,
|
||||
GATEWAY,
|
||||
} from "./agent.ts";
|
||||
import { mkdir, writeFile, appendFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
const EXECUTOR_MODEL = "mistral:latest";
|
||||
const REVIEWER_MODEL = "qwen2.5:latest";
|
||||
const DRAFT_MODEL = "qwen2.5:latest"; // artifact generation; short outputs
|
||||
const MAX_TURNS = 14;
|
||||
const MAX_CONSECUTIVE_DRIFTS = 3;
|
||||
const WORKERS_INDEX = "workers_500k_v1";
|
||||
const WORKERS_DATASET = "workers_500k";
|
||||
|
||||
// =================== Event + scenario types ===================
|
||||
|
||||
type EventKind = "baseline_fill" | "recurring" | "expansion" | "emergency" | "misplacement";
|
||||
|
||||
interface FillEvent {
|
||||
kind: EventKind;
|
||||
at: string; // display label like "08:00"
|
||||
role: string;
|
||||
count: number;
|
||||
city: string;
|
||||
state: string;
|
||||
shift_start?: string; // "08:00 AM" for SMS/email drafts
|
||||
scenario_note?: string; // extra context the agents should know
|
||||
deadline?: string; // emergency events carry this, shown to reviewer
|
||||
exclude_worker_ids?: string[]; // misplacement: the lost worker
|
||||
replaces_event?: string; // misplacement back-ref for reporting
|
||||
}
|
||||
|
||||
interface ScenarioSpec {
|
||||
client: string;
|
||||
date: string;
|
||||
events: FillEvent[];
|
||||
}
|
||||
|
||||
interface EventResult {
|
||||
event: FillEvent;
|
||||
ok: boolean;
|
||||
fills: Fill[];
|
||||
turns: number;
|
||||
duration_secs: number;
|
||||
error?: string;
|
||||
gap_signals: string[]; // pulled into the cross-event gap report
|
||||
sources_first_score?: number;
|
||||
sources_last_score?: number;
|
||||
pool_size?: number; // sql_matches from the first hybrid_search
|
||||
playbook_citations?: string[];
|
||||
}
|
||||
|
||||
interface RosterEntry {
|
||||
worker_id: string;
|
||||
name: string;
|
||||
booked_for: string; // event at-label
|
||||
role: string;
|
||||
city: string;
|
||||
state: string;
|
||||
status: "confirmed" | "no_show" | "rebooked_elsewhere";
|
||||
}
|
||||
|
||||
interface ScenarioContext {
|
||||
spec: ScenarioSpec;
|
||||
out_dir: string;
|
||||
roster: RosterEntry[];
|
||||
results: EventResult[];
|
||||
gap_signals: Array<{ event: string; category: string; detail: string }>;
|
||||
}
|
||||
|
||||
// =================== Default scenario ===================
|
||||
|
||||
const DEFAULT_SCENARIO: ScenarioSpec = {
|
||||
client: "Riverfront Steel",
|
||||
date: "2026-04-21",
|
||||
events: [
|
||||
{
|
||||
kind: "baseline_fill",
|
||||
at: "08:00",
|
||||
role: "Warehouse Associate",
|
||||
count: 3,
|
||||
city: "Toledo",
|
||||
state: "OH",
|
||||
shift_start: "08:00 AM",
|
||||
scenario_note: "Regular Monday morning shift, 8-hour.",
|
||||
},
|
||||
{
|
||||
kind: "recurring",
|
||||
at: "10:30",
|
||||
role: "Machine Operator",
|
||||
count: 2,
|
||||
city: "Toledo",
|
||||
state: "OH",
|
||||
shift_start: "11:00 AM",
|
||||
scenario_note: "Recurring Tuesday/Thursday slot — prior workers may still be available.",
|
||||
},
|
||||
{
|
||||
kind: "expansion",
|
||||
at: "12:15",
|
||||
role: "Forklift Operator",
|
||||
count: 5,
|
||||
city: "Toledo",
|
||||
state: "OH",
|
||||
shift_start: "01:00 PM",
|
||||
scenario_note: "New warehouse location opening, five-worker team needed.",
|
||||
},
|
||||
{
|
||||
kind: "emergency",
|
||||
at: "14:00",
|
||||
role: "Loader",
|
||||
count: 4,
|
||||
city: "Toledo",
|
||||
state: "OH",
|
||||
shift_start: "04:00 PM same day",
|
||||
deadline: "16:00",
|
||||
scenario_note: "Walkoff incident — replacement crew needed by 16:00 sharp.",
|
||||
},
|
||||
{
|
||||
kind: "misplacement",
|
||||
at: "15:45",
|
||||
role: "Warehouse Associate",
|
||||
count: 1,
|
||||
city: "Toledo",
|
||||
state: "OH",
|
||||
shift_start: "remainder of 08:00 shift",
|
||||
scenario_note: "One worker from the 08:00 fill didn't show; rebuild the gap.",
|
||||
replaces_event: "08:00",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// =================== Low-level helpers shared across events ===================
|
||||
|
||||
async function httpJson<T>(url: string, body?: any): Promise<T> {
|
||||
const res = await fetch(url, {
|
||||
method: body ? "POST" : "GET",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
if (!res.ok) throw new Error(`${res.status} ${await res.text()}`);
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
|
||||
function fmt(e: LogEntry): string {
|
||||
const tag = ` [t${e.turn.toString().padStart(2, "0")} ${e.role.padEnd(8)} ${e.kind.padEnd(14)}]`;
|
||||
const c = e.content ?? {};
|
||||
const trim = (s: any, n: number) => String(s ?? "").slice(0, n);
|
||||
if (e.kind === "tool_call") return `${tag} ${c.tool}(${JSON.stringify(c.args ?? {}).slice(0, 60)}) — ${trim(c.rationale, 40)}`;
|
||||
if (e.kind === "tool_result") {
|
||||
if (c.error) return `${tag} ERROR ${c.error}`;
|
||||
const rows = c?.rows?.length ?? c?.sources?.length ?? undefined;
|
||||
return `${tag} ${rows !== undefined ? `rows=${rows}` : JSON.stringify(c).slice(0, 60)}`;
|
||||
}
|
||||
if (e.kind === "critique") return `${tag} verdict=${c.verdict} — ${trim(c.notes, 50)}`;
|
||||
if (e.kind === "propose_done") return `${tag} ${c.fills?.length ?? 0} fills: ${(c.fills ?? []).map((f: Fill) => f.name).join(", ")}`;
|
||||
if (e.kind === "consensus_done") return `${tag} ✓`;
|
||||
if (e.kind === "plan") return `${tag} ${c.steps?.length ?? 0} steps`;
|
||||
if (e.kind === "error") return `${tag} ${c.message ?? c}`;
|
||||
return `${tag} ${JSON.stringify(c).slice(0, 70)}`;
|
||||
}
|
||||
|
||||
async function executeToolCall(name: string, args: Record<string, any>): Promise<any> {
|
||||
if (name === "hybrid_search") {
|
||||
const { sql_filter, question, index_name, k } = args;
|
||||
if (!sql_filter || !question || !index_name) {
|
||||
throw new Error(`hybrid_search needs sql_filter + question + index_name, got ${JSON.stringify(args)}`);
|
||||
}
|
||||
// Every fill event uses the playbook_memory boost — that's the point
|
||||
// of the run-as-a-whole: earlier events seed later ones.
|
||||
return httpJson(`${GATEWAY}/vectors/hybrid`, {
|
||||
sql_filter, question, index_name,
|
||||
top_k: k ?? 10, generate: false,
|
||||
use_playbook_memory: true,
|
||||
playbook_memory_k: 10,
|
||||
});
|
||||
}
|
||||
if (name === "sql") {
|
||||
const { query } = args;
|
||||
if (!query || typeof query !== "string") throw new Error(`sql needs query string`);
|
||||
if (!/^\s*SELECT/i.test(query)) throw new Error(`sql allows SELECT only`);
|
||||
return sqlQuery(query);
|
||||
}
|
||||
return callTool(name, args);
|
||||
}
|
||||
|
||||
// =================== Core fill loop — one event, one consensus ===================
|
||||
|
||||
interface AgentFillOutcome {
|
||||
fills: Fill[];
|
||||
approach: string;
|
||||
turns: number;
|
||||
duration_secs: number;
|
||||
log: LogEntry[];
|
||||
first_sql_matches?: number;
|
||||
first_pool_first_score?: number;
|
||||
first_pool_last_score?: number;
|
||||
playbook_citations: string[];
|
||||
}
|
||||
|
||||
async function runAgentFill(
|
||||
task: TaskSpec,
|
||||
extra_guidance: string,
|
||||
exclude_worker_ids: string[],
|
||||
): Promise<AgentFillOutcome> {
|
||||
const t0 = Date.now();
|
||||
const log: LogEntry[] = [];
|
||||
let turn = 0;
|
||||
let consecutiveDrifts = 0;
|
||||
let sealed: { fills: Fill[]; approach: string } | null = null;
|
||||
let first_sql_matches: number | undefined;
|
||||
let first_pool_first: number | undefined;
|
||||
let first_pool_last: number | undefined;
|
||||
const playbook_citations = new Set<string>();
|
||||
|
||||
const append = (e: Omit<LogEntry, "at">): LogEntry => {
|
||||
const full: LogEntry = { ...e, at: new Date().toISOString() };
|
||||
log.push(full);
|
||||
console.log(fmt(full));
|
||||
return full;
|
||||
};
|
||||
|
||||
// Build executor prompt with the scenario-specific guidance + exclusions
|
||||
// injected as an extra block. Reuses the base prompt so drift detection
|
||||
// and output-shape rules are unchanged.
|
||||
const withExtras = (base: string): string => {
|
||||
let addon = "";
|
||||
if (extra_guidance) addon += `\n\nEVENT-SPECIFIC GUIDANCE:\n${extra_guidance}`;
|
||||
if (exclude_worker_ids.length > 0) {
|
||||
addon += `\n\nEXCLUDE these workers (already booked / unavailable today): ${exclude_worker_ids.join(", ")}\nIf your tool results include them, skip them — never propose them.`;
|
||||
}
|
||||
return base + addon;
|
||||
};
|
||||
|
||||
while (turn < MAX_TURNS && !sealed) {
|
||||
turn += 1;
|
||||
|
||||
const execRaw = await generate(
|
||||
EXECUTOR_MODEL,
|
||||
withExtras(executorPrompt(task, log)),
|
||||
{ temperature: 0.2, max_tokens: 600 },
|
||||
);
|
||||
let execAction: Action;
|
||||
try {
|
||||
execAction = parseAction(execRaw, "executor");
|
||||
} catch (e) {
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "error",
|
||||
content: { message: (e as Error).message, raw: execRaw.slice(0, 300) } });
|
||||
throw e;
|
||||
}
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL,
|
||||
kind: execAction.kind as any, content: execAction });
|
||||
|
||||
if (execAction.kind === "tool_call") {
|
||||
try {
|
||||
const result = await executeToolCall(execAction.tool, execAction.args);
|
||||
// Filter tool results to enforce the exclusion list — defense in
|
||||
// depth since the prompt alone isn't enough for weak models.
|
||||
const filtered = maskExclusions(result, exclude_worker_ids);
|
||||
// Capture the first hybrid_search pool stats for gap detection.
|
||||
if (execAction.tool === "hybrid_search" && first_sql_matches === undefined) {
|
||||
first_sql_matches = (filtered as any).sql_matches;
|
||||
const sources = (filtered as any).sources ?? [];
|
||||
if (sources.length > 0) {
|
||||
first_pool_first = sources[0].score;
|
||||
first_pool_last = sources[sources.length - 1].score;
|
||||
}
|
||||
}
|
||||
const trimmed = trimResult(filtered);
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL,
|
||||
kind: "tool_result", content: trimmed });
|
||||
|
||||
// Accumulate playbook citations from any hybrid result that
|
||||
// carried them — the scenario-level report needs them.
|
||||
if (Array.isArray((filtered as any).sources)) {
|
||||
for (const s of (filtered as any).sources) {
|
||||
for (const c of s.playbook_citations ?? []) {
|
||||
playbook_citations.add(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
append({ turn, role: "executor", model: EXECUTOR_MODEL, kind: "tool_result",
|
||||
content: { error: (e as Error).message, tool: execAction.tool } });
|
||||
consecutiveDrifts += 1;
|
||||
if (consecutiveDrifts >= MAX_CONSECUTIVE_DRIFTS) {
|
||||
throw new Error(`aborted — ${MAX_CONSECUTIVE_DRIFTS} consecutive tool errors`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const revRaw = await generate(
|
||||
REVIEWER_MODEL,
|
||||
withExtras(reviewerPrompt(task, log)),
|
||||
{ temperature: 0.1, max_tokens: 400 },
|
||||
);
|
||||
let revAction: Action;
|
||||
try {
|
||||
revAction = parseAction(revRaw, "reviewer");
|
||||
} catch (e) {
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "error",
|
||||
content: { message: (e as Error).message, raw: revRaw.slice(0, 300) } });
|
||||
throw e;
|
||||
}
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL,
|
||||
kind: "critique", content: revAction });
|
||||
|
||||
if (revAction.kind !== "critique") throw new Error(`reviewer emitted non-critique: ${revAction.kind}`);
|
||||
|
||||
if (revAction.verdict === "drift") {
|
||||
consecutiveDrifts += 1;
|
||||
if (consecutiveDrifts >= MAX_CONSECUTIVE_DRIFTS) {
|
||||
throw new Error(`aborted — ${MAX_CONSECUTIVE_DRIFTS} consecutive drift flags`);
|
||||
}
|
||||
} else {
|
||||
consecutiveDrifts = 0;
|
||||
}
|
||||
|
||||
if (execAction.kind === "propose_done" && revAction.verdict === "approve_done") {
|
||||
if (execAction.fills.length !== task.target_count) {
|
||||
throw new Error(`consensus malformed — ${execAction.fills.length} fills vs target ${task.target_count}`);
|
||||
}
|
||||
// Enforce exclusion at seal time too, in case the models ignored
|
||||
// both prompt + tool-result filtering.
|
||||
for (const f of execAction.fills) {
|
||||
if (exclude_worker_ids.includes(f.candidate_id)) {
|
||||
throw new Error(`consensus proposed excluded worker ${f.candidate_id}`);
|
||||
}
|
||||
}
|
||||
append({ turn, role: "reviewer", model: REVIEWER_MODEL, kind: "consensus_done",
|
||||
content: { fills: execAction.fills } });
|
||||
sealed = { fills: execAction.fills, approach: execAction.rationale ?? "multi-agent hybrid" };
|
||||
}
|
||||
}
|
||||
|
||||
if (!sealed) throw new Error(`no consensus after ${MAX_TURNS} turns`);
|
||||
|
||||
return {
|
||||
fills: sealed.fills,
|
||||
approach: sealed.approach,
|
||||
turns: turn,
|
||||
duration_secs: (Date.now() - t0) / 1000,
|
||||
log,
|
||||
first_sql_matches,
|
||||
first_pool_first_score: first_pool_first,
|
||||
first_pool_last_score: first_pool_last,
|
||||
playbook_citations: Array.from(playbook_citations),
|
||||
};
|
||||
}
|
||||
|
||||
function maskExclusions(result: any, exclude: string[]): any {
|
||||
if (exclude.length === 0) return result;
|
||||
if (Array.isArray(result.sources)) {
|
||||
return { ...result, sources: result.sources.filter((s: any) => !exclude.includes(s.doc_id)) };
|
||||
}
|
||||
if (Array.isArray(result.rows)) {
|
||||
return { ...result, rows: result.rows.filter((r: any) => {
|
||||
const id = r.worker_id ?? r.doc_id;
|
||||
return id === undefined || !exclude.includes(String(id));
|
||||
}) };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function trimResult(r: any): any {
|
||||
if (r && Array.isArray(r.sources)) {
|
||||
return { ...r, sources: r.sources.slice(0, 20), _trimmed: r.sources.length > 20 ? `${r.sources.length - 20} more` : undefined };
|
||||
}
|
||||
if (r && Array.isArray(r.rows)) {
|
||||
return { ...r, rows: r.rows.slice(0, 20), _trimmed: r.rows.length > 20 ? `${r.rows.length - 20} more` : undefined };
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// =================== Per-event guidance strings ===================
|
||||
|
||||
function guidanceFor(event: FillEvent, ctx: ScenarioContext): string {
|
||||
switch (event.kind) {
|
||||
case "baseline_fill":
|
||||
return `Standard Monday fill. Client ${ctx.spec.client}. Shift starts ${event.shift_start ?? "at start time"}. Take the top candidates by semantic match and availability.`;
|
||||
case "recurring":
|
||||
return `RECURRING slot — ${ctx.spec.client} runs this shift every Tues/Thurs. If playbook_memory surfaces candidates endorsed by past similar fills (you'll see 'cites' on hybrid sources), those are the preferred workers. Shift starts ${event.shift_start ?? "at start time"}.`;
|
||||
case "expansion":
|
||||
return `EXPANSION at ${ctx.spec.client}. New location, ${event.count}-worker team needed at once — search broadly and prefer workers with team/collaboration signals (engagement, communications scores). Shift starts ${event.shift_start ?? "at start time"}.`;
|
||||
case "emergency":
|
||||
return `EMERGENCY walkoff — ${ctx.spec.client} needs ${event.count} ${event.role}s BY ${event.deadline ?? "end of day"}. Prioritize availability over perfect skill match. A good-enough worker who can report today beats a perfect worker who can't.`;
|
||||
case "misplacement":
|
||||
return `MISPLACEMENT refill. A worker from the 08:00 shift no-showed. You must replace them WITHOUT proposing the same worker or anyone already booked today (see EXCLUDE list). Shift is ${event.shift_start ?? "in progress"} so speed matters.`;
|
||||
}
|
||||
}
|
||||
|
||||
// =================== Artifact generation ===================
|
||||
|
||||
interface ArtifactBundle {
|
||||
sms: string;
|
||||
email: string;
|
||||
}
|
||||
|
||||
// One Ollama call per event for SMS (to the filled workers) + one for
|
||||
// the client email. Short outputs, low temperature — these are drafts,
|
||||
// not creative writing.
|
||||
async function generateArtifacts(event: FillEvent, outcome: AgentFillOutcome, ctx: ScenarioContext): Promise<ArtifactBundle> {
|
||||
const smsPrompt = `Generate short, friendly, professional SMS messages to confirm a shift for each worker. ONE message per worker. Format as:
|
||||
|
||||
TO: {Name}
|
||||
{message body under 180 chars}
|
||||
|
||||
---
|
||||
|
||||
Details:
|
||||
- Client: ${ctx.spec.client}
|
||||
- Role: ${event.role}
|
||||
- Location: ${event.city}, ${event.state}
|
||||
- Shift starts: ${event.shift_start ?? "TBD"}
|
||||
- Scenario: ${event.scenario_note ?? ""}
|
||||
|
||||
Workers to message:
|
||||
${outcome.fills.map(f => `- ${f.name} (id ${f.candidate_id})`).join("\n")}
|
||||
|
||||
Respond with only the message blocks, separated by "---". No commentary.`;
|
||||
|
||||
const emailPrompt = `Generate a short professional email confirmation to the staffing client.
|
||||
|
||||
TO: staffing@${ctx.spec.client.toLowerCase().replace(/ /g, "")}.example
|
||||
FROM: dispatch@lakehouse.example
|
||||
SUBJECT: (3-word subject)
|
||||
|
||||
Body (4-6 lines max). Be specific about:
|
||||
- Number of workers filled (${outcome.fills.length} of ${event.count})
|
||||
- Roles: ${event.role}
|
||||
- Names filled
|
||||
- Shift start: ${event.shift_start ?? "TBD"}
|
||||
- Any scenario flag: ${event.scenario_note ?? "(none)"}
|
||||
|
||||
Workers:
|
||||
${outcome.fills.map(f => `- ${f.name} (${f.reason.slice(0, 60)})`).join("\n")}
|
||||
|
||||
Respond with only the email. No commentary.`;
|
||||
|
||||
const [sms, email] = await Promise.all([
|
||||
generate(DRAFT_MODEL, smsPrompt, { temperature: 0.3, max_tokens: 500 }),
|
||||
generate(DRAFT_MODEL, emailPrompt, { temperature: 0.3, max_tokens: 400 }),
|
||||
]);
|
||||
|
||||
return { sms: sms.trim(), email: email.trim() };
|
||||
}
|
||||
|
||||
// =================== Per-event runner ===================
|
||||
|
||||
async function runEvent(event: FillEvent, ctx: ScenarioContext): Promise<EventResult> {
|
||||
console.log(`\n════════ ${event.at} — ${event.kind.toUpperCase()}: fill ${event.count}× ${event.role} in ${event.city}, ${event.state} ════════`);
|
||||
|
||||
const t0 = Date.now();
|
||||
|
||||
// Build the task spec the agent loop expects.
|
||||
const task: TaskSpec = {
|
||||
id: `${ctx.spec.date}-${event.at.replace(":", "")}-${event.kind}`,
|
||||
operation: `fill: ${event.role} x${event.count} in ${event.city}, ${event.state}`,
|
||||
target_role: event.role,
|
||||
target_count: event.count,
|
||||
target_city: event.city,
|
||||
target_state: event.state,
|
||||
approach_hint: `hybrid search against ${WORKERS_INDEX} for ${event.kind}`,
|
||||
};
|
||||
|
||||
// Exclusion set: everyone already in today's roster + any explicit
|
||||
// exclusions from the event spec.
|
||||
const excludeIds = [
|
||||
...ctx.roster
|
||||
.filter(r => r.status === "confirmed")
|
||||
.map(r => r.worker_id),
|
||||
...(event.exclude_worker_ids ?? []),
|
||||
];
|
||||
|
||||
const gap_signals: string[] = [];
|
||||
let outcome: AgentFillOutcome;
|
||||
try {
|
||||
outcome = await runAgentFill(task, guidanceFor(event, ctx), excludeIds);
|
||||
} catch (e) {
|
||||
return {
|
||||
event,
|
||||
ok: false,
|
||||
fills: [],
|
||||
turns: 0,
|
||||
duration_secs: (Date.now() - t0) / 1000,
|
||||
error: (e as Error).message,
|
||||
gap_signals: [`drift_or_tool: ${(e as Error).message}`],
|
||||
};
|
||||
}
|
||||
|
||||
// Resolve worker_ids via SQL so the roster has stable IDs (models
|
||||
// sometimes return names-only). Best-effort — if name lookup finds
|
||||
// zero or many matches, we flag a gap.
|
||||
const resolved = await resolveWorkerIds(outcome.fills, event);
|
||||
|
||||
// Roster double-book check.
|
||||
for (const r of resolved) {
|
||||
const conflict = ctx.roster.find(e => e.worker_id === r.worker_id && e.status === "confirmed");
|
||||
if (conflict) {
|
||||
gap_signals.push(`double_book: ${r.worker_id} ${r.name} already booked for ${conflict.booked_for}`);
|
||||
}
|
||||
ctx.roster.push({
|
||||
worker_id: r.worker_id,
|
||||
name: r.name,
|
||||
booked_for: event.at,
|
||||
role: event.role,
|
||||
city: event.city,
|
||||
state: event.state,
|
||||
status: "confirmed",
|
||||
});
|
||||
}
|
||||
|
||||
// Pool-size signal (Gap 1 — supply).
|
||||
const supply_threshold = event.count * 3;
|
||||
if ((outcome.first_sql_matches ?? 0) < supply_threshold) {
|
||||
gap_signals.push(
|
||||
`supply: only ${outcome.first_sql_matches} candidates for ${event.count}× ${event.role} in ${event.city} (< ${supply_threshold}, our 3× comfort margin)`
|
||||
);
|
||||
}
|
||||
|
||||
// Score-spread signal (Gap 2 — embedding).
|
||||
const spread = (outcome.first_pool_first_score ?? 0) - (outcome.first_pool_last_score ?? 0);
|
||||
if (spread > 0 && spread < 0.02) {
|
||||
gap_signals.push(
|
||||
`embedding: top-K score spread ${spread.toFixed(3)} < 0.02 — model struggles to differentiate`
|
||||
);
|
||||
}
|
||||
|
||||
// Generate artifacts (SMS + email) — fail-soft; artifacts are cosmetic
|
||||
// relative to the consensus itself.
|
||||
let bundle: ArtifactBundle | null = null;
|
||||
try {
|
||||
bundle = await generateArtifacts(event, { ...outcome, fills: resolved }, ctx);
|
||||
await appendFile(join(ctx.out_dir, "sms.md"),
|
||||
`\n## ${event.at} ${event.kind} — ${event.role} x${event.count} in ${event.city}, ${event.state}\n\n${bundle.sms}\n`);
|
||||
await appendFile(join(ctx.out_dir, "emails.md"),
|
||||
`\n## ${event.at} ${event.kind} — ${event.role} x${event.count}\n\n${bundle.email}\n`);
|
||||
} catch (e) {
|
||||
gap_signals.push(`artifact: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
// Dispatch log (structured).
|
||||
await appendFile(join(ctx.out_dir, "dispatch.jsonl"),
|
||||
JSON.stringify({
|
||||
at: event.at,
|
||||
kind: event.kind,
|
||||
operation: task.operation,
|
||||
fills: resolved,
|
||||
turns: outcome.turns,
|
||||
duration_secs: outcome.duration_secs,
|
||||
pool_size: outcome.first_sql_matches,
|
||||
playbook_citations: outcome.playbook_citations,
|
||||
}) + "\n");
|
||||
|
||||
// Always seed playbook_memory after a sealed fill — keep the learning
|
||||
// loop tight across the whole day so recurring/misplacement events
|
||||
// later in the run benefit from earlier ones.
|
||||
try {
|
||||
await httpJson(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
operation: task.operation,
|
||||
approach: outcome.approach || `${event.kind} → hybrid search`,
|
||||
context: `client=${ctx.spec.client} scenario=${event.kind} shift=${event.shift_start ?? "tbd"}`,
|
||||
endorsed_names: resolved.map(r => r.name),
|
||||
append: true,
|
||||
});
|
||||
} catch (e) {
|
||||
gap_signals.push(`write_through: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
event,
|
||||
ok: true,
|
||||
fills: outcome.fills,
|
||||
turns: outcome.turns,
|
||||
duration_secs: outcome.duration_secs,
|
||||
gap_signals,
|
||||
sources_first_score: outcome.first_pool_first_score,
|
||||
sources_last_score: outcome.first_pool_last_score,
|
||||
pool_size: outcome.first_sql_matches,
|
||||
playbook_citations: outcome.playbook_citations,
|
||||
};
|
||||
}
|
||||
|
||||
// =================== Worker ID resolution ===================
|
||||
|
||||
// Models emit candidate_ids or names in propose_done. Some return the
|
||||
// W500K-XXX doc_id, others just the name, others a random tag. Resolve
|
||||
// to canonical (worker_id, name) via SQL so the roster is reliable.
|
||||
async function resolveWorkerIds(fills: Fill[], event: FillEvent): Promise<Fill[]> {
|
||||
const resolved: Fill[] = [];
|
||||
for (const f of fills) {
|
||||
// Case 1: candidate_id looks like W500K-NNN — accept as-is.
|
||||
if (/^W500K-\d+$/.test(f.candidate_id)) {
|
||||
resolved.push(f);
|
||||
continue;
|
||||
}
|
||||
// Case 2: candidate_id is a bare integer — promote to W500K-N.
|
||||
if (/^\d+$/.test(f.candidate_id)) {
|
||||
resolved.push({ ...f, candidate_id: `W500K-${f.candidate_id}` });
|
||||
continue;
|
||||
}
|
||||
// Case 3: look up by (name, city, state). Take the first match.
|
||||
const q = `SELECT worker_id FROM ${WORKERS_DATASET} WHERE name = '${f.name.replace(/'/g, "''")}' AND city = '${event.city.replace(/'/g, "''")}' AND state = '${event.state.replace(/'/g, "''")}' LIMIT 1`;
|
||||
try {
|
||||
const r = await sqlQuery(q);
|
||||
if (r.rows && r.rows.length > 0) {
|
||||
resolved.push({ ...f, candidate_id: `W500K-${r.rows[0].worker_id}` });
|
||||
} else {
|
||||
// No match — keep the fill but leave candidate_id as-is; the
|
||||
// gap report will flag it.
|
||||
resolved.push(f);
|
||||
}
|
||||
} catch {
|
||||
resolved.push(f);
|
||||
}
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
// =================== EOD gap report ===================
|
||||
|
||||
async function writeRetrospective(ctx: ScenarioContext): Promise<void> {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Scenario retrospective — ${ctx.spec.client}, ${ctx.spec.date}`);
|
||||
lines.push("");
|
||||
lines.push(`Executor: \`${EXECUTOR_MODEL}\` Reviewer: \`${REVIEWER_MODEL}\` Draft: \`${DRAFT_MODEL}\``);
|
||||
lines.push("");
|
||||
|
||||
// --- Per-event summary ---
|
||||
lines.push("## Events");
|
||||
lines.push("");
|
||||
lines.push("| At | Kind | Role / Count | Pool | Fills | Turns | Dur(s) | Cites | Gaps |");
|
||||
lines.push("|---|---|---|---|---|---|---|---|---|");
|
||||
for (const r of ctx.results) {
|
||||
const status = r.ok ? "✓" : "✗";
|
||||
lines.push(
|
||||
`| ${r.event.at} | ${r.event.kind} | ${r.event.role} × ${r.event.count} | ${r.pool_size ?? "-"} | ${status} ${r.fills.length} | ${r.turns} | ${r.duration_secs.toFixed(1)} | ${r.playbook_citations?.length ?? 0} | ${r.gap_signals.length} |`
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
// --- Roster ---
|
||||
lines.push("## Final roster");
|
||||
lines.push("");
|
||||
lines.push("| Worker | Booked | Role | City, ST | Status |");
|
||||
lines.push("|---|---|---|---|---|");
|
||||
for (const e of ctx.roster) {
|
||||
lines.push(`| ${e.worker_id} ${e.name} | ${e.booked_for} | ${e.role} | ${e.city}, ${e.state} | ${e.status} |`);
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
// --- Gap analysis by category ---
|
||||
const bycat: Record<string, string[]> = {};
|
||||
for (const g of ctx.gap_signals) {
|
||||
if (!bycat[g.category]) bycat[g.category] = [];
|
||||
bycat[g.category].push(`**${g.event}** — ${g.detail}`);
|
||||
}
|
||||
|
||||
// Add cross-event categories computed here:
|
||||
// Gap 3 — fairness (Gini-lite on roster)
|
||||
const bookedIds = ctx.roster.filter(r => r.status === "confirmed").map(r => r.worker_id);
|
||||
const counts = new Map<string, number>();
|
||||
for (const id of bookedIds) counts.set(id, (counts.get(id) ?? 0) + 1);
|
||||
const multis = [...counts.entries()].filter(([_, n]) => n > 1);
|
||||
if (multis.length > 0) {
|
||||
bycat["fairness"] = bycat["fairness"] ?? [];
|
||||
for (const [id, n] of multis) {
|
||||
const name = ctx.roster.find(r => r.worker_id === id)?.name ?? id;
|
||||
bycat["fairness"].push(`_cross-event_ — ${name} (${id}) booked ${n} times today`);
|
||||
}
|
||||
}
|
||||
|
||||
// Gap 5 — tool errors already captured per-event via gap_signals.
|
||||
|
||||
// Gap 6 — write-through coverage: compare # events vs # new playbook_memory entries.
|
||||
try {
|
||||
const stats = await httpJson<any>(`${GATEWAY}/vectors/playbook_memory/stats`);
|
||||
bycat["write_through_audit"] = bycat["write_through_audit"] ?? [];
|
||||
bycat["write_through_audit"].push(`_post-run_ — playbook_memory has ${stats.entries} entries (ran ${ctx.results.length} events, expected ≥ ${ctx.results.filter(r => r.ok).length} new entries from this run)`);
|
||||
} catch { /* non-fatal */ }
|
||||
|
||||
lines.push("## Gap signals");
|
||||
lines.push("");
|
||||
if (Object.keys(bycat).length === 0) {
|
||||
lines.push("_None surfaced — either everything worked or detection is under-tuned._");
|
||||
} else {
|
||||
for (const [cat, items] of Object.entries(bycat)) {
|
||||
lines.push(`### ${cat}`);
|
||||
for (const item of items) lines.push(`- ${item}`);
|
||||
lines.push("");
|
||||
}
|
||||
}
|
||||
|
||||
// --- Narrative summary ---
|
||||
lines.push("## Narrative");
|
||||
lines.push("");
|
||||
lines.push(`- ${ctx.results.filter(r => r.ok).length}/${ctx.results.length} events reached consensus.`);
|
||||
lines.push(`- Final roster: ${ctx.roster.length} bookings across ${new Set(ctx.roster.map(r => r.worker_id)).size} distinct workers.`);
|
||||
const totalCites = ctx.results.reduce((a, r) => a + (r.playbook_citations?.length ?? 0), 0);
|
||||
lines.push(`- Playbook citations across the day: ${totalCites} (proof the feedback loop fired across events).`);
|
||||
const droppedEvents = ctx.results.filter(r => !r.ok);
|
||||
if (droppedEvents.length > 0) {
|
||||
lines.push(`- Dropped events: ${droppedEvents.map(r => r.event.at + " " + r.event.kind).join(", ")}.`);
|
||||
}
|
||||
|
||||
await writeFile(join(ctx.out_dir, "report.md"), lines.join("\n"));
|
||||
console.log(`\n✓ report → ${join(ctx.out_dir, "report.md")}`);
|
||||
}
|
||||
|
||||
// =================== Main driver ===================
|
||||
|
||||
async function main() {
|
||||
const specPath = process.argv[2];
|
||||
const spec: ScenarioSpec = specPath
|
||||
? JSON.parse(await Bun.file(specPath).text())
|
||||
: DEFAULT_SCENARIO;
|
||||
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
||||
const out_dir = join("tests/multi-agent/playbooks", `scenario-${stamp}`);
|
||||
await mkdir(out_dir, { recursive: true });
|
||||
|
||||
const ctx: ScenarioContext = {
|
||||
spec,
|
||||
out_dir,
|
||||
roster: [],
|
||||
results: [],
|
||||
gap_signals: [],
|
||||
};
|
||||
|
||||
// Initialize output files
|
||||
await writeFile(join(out_dir, "sms.md"), `# SMS drafts — ${spec.client}, ${spec.date}\n`);
|
||||
await writeFile(join(out_dir, "emails.md"), `# Client emails — ${spec.client}, ${spec.date}\n`);
|
||||
await writeFile(join(out_dir, "dispatch.jsonl"), "");
|
||||
|
||||
console.log(`▶ scenario: ${spec.client}, ${spec.date}, ${spec.events.length} events`);
|
||||
console.log(`▶ out: ${out_dir}\n`);
|
||||
|
||||
for (const event of spec.events) {
|
||||
// Expand misplacement-style exclusions from the current roster: it
|
||||
// wants to replace a worker from a prior event, so grab everyone
|
||||
// booked at that at-label and add as exclusions.
|
||||
if (event.kind === "misplacement" && event.replaces_event) {
|
||||
const priorBooked = ctx.roster
|
||||
.filter(r => r.booked_for === event.replaces_event && r.status === "confirmed")
|
||||
.map(r => r.worker_id);
|
||||
if (priorBooked.length > 0) {
|
||||
// Pick one arbitrarily to mark as no_show — in a real system the
|
||||
// external signal would pick. For the test, first one works.
|
||||
const lost = priorBooked[0];
|
||||
const lostEntry = ctx.roster.find(r => r.worker_id === lost);
|
||||
if (lostEntry) {
|
||||
lostEntry.status = "no_show";
|
||||
console.log(` (misplacement: marking ${lost} ${lostEntry.name} as no-show)`);
|
||||
}
|
||||
// Exclude all prior bookings so the refill doesn't pick anyone
|
||||
// already scheduled for today.
|
||||
event.exclude_worker_ids = priorBooked;
|
||||
}
|
||||
}
|
||||
|
||||
const result = await runEvent(event, ctx);
|
||||
ctx.results.push(result);
|
||||
for (const s of result.gap_signals) {
|
||||
const [category, ...rest] = s.split(":");
|
||||
ctx.gap_signals.push({ event: event.at, category: category.trim(), detail: rest.join(":").trim() });
|
||||
}
|
||||
|
||||
// Small breather to not hammer Ollama on back-to-back runs.
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
}
|
||||
|
||||
// Persist structured state for forensics.
|
||||
await writeFile(join(out_dir, "roster.json"), JSON.stringify(ctx.roster, null, 2));
|
||||
await writeFile(join(out_dir, "results.json"), JSON.stringify(ctx.results, null, 2));
|
||||
|
||||
await writeRetrospective(ctx);
|
||||
|
||||
const okCount = ctx.results.filter(r => r.ok).length;
|
||||
if (okCount < ctx.results.length) {
|
||||
console.log(`\n⚠ ${okCount}/${ctx.results.length} events succeeded. See ${out_dir}/report.md for gaps.`);
|
||||
process.exit(2);
|
||||
}
|
||||
console.log(`\n✓ ${okCount}/${ctx.results.length} events succeeded. See ${out_dir}/report.md.`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error(`\n✗ scenario driver crashed: ${(e as Error).message}`);
|
||||
console.error((e as Error).stack);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user