Coherence pass — PRD/PHASES updates, config snapshot wired, unit tests
J flagged the audit: "make sure everything flows coherently, no pseudocode or unnecessary patches or ignoring any particular part of what we built." This is that pass. PRD.md updates: - Phase 19 refinement block — geo-filter + role-prefilter WIRED with citation density numbers (0.32 → 1.38, and 2 → 28 on same scenario). - Phase 20 rewrite — mistral dropped, qwen3.5 + qwen3 local hot path, think:false as the key mechanical finding, kimi-k2.6 upgrade path. - Phase 21 status block — think plumbing + cloud executor routing added after original commit. - Phase 22 item B (cloud rescue) — pivot sanitizer, rescue verified 1/3 on stress_01. - Phase 23 NEW — staffer identity + tool_level + competence-weighted retrieval + kb_staffer_report. Auto-discovered worker labels called out with real numbers (Rachel Lewis 12× across 4 staffers). - Phase 24 NEW — Observer/Autotune integration gap DOCUMENTED, not fixed. Observer has been idle at 0 ops for 3600+ cycles because scenarios hit gateway:3100 directly, bypassing MCP:3700 which the observer wraps. This is the honest "we're not using it in these tests" signal J surfaced. Fix deferred; gap visible now. PHASES.md: - Appended Phases 20-23 as checked, Phase 24 as unchecked gap. - Updated footer count: 102 unit tests across all layers. - Latest line updated with 14× citation lift + 46.4pt tool-asymmetry finding. scenario.ts: - snapshotConfig() was defined but never called. Now fires at every scenario start with a stable sha256 hash over the active model set + tool_level + cloud flags. config_snapshots.jsonl finally populates, which the error_corrections diff path needs to work correctly. kb.test.ts (new): 4 signature invariant tests — stability across unrelated fields (date, contract, staffer), sensitivity to role/city/ count changes, digest shape. All pass under `bun test`. service.rs: 6 Rust extractor tests for extract_target_geo + extract_target_role — basic, missing-state-returns-none, word boundary (civilian != city), multi-word role, absent role, quoted value parse. All pass under `cargo test -p vectord --lib extractor_tests`. Dangling items now honestly documented rather than silently pending: - Chunking cache (config/models.json SPEC, not wired) — flagged - Playbook versioning (SPEC, not wired) — flagged - Observer integration (WIRED but disconnected) — new Phase 24
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import { test, expect } from "bun:test";
|
||||
import { computeSignature, specDigest } from "./kb.ts";
|
||||
|
||||
// kb signature invariants — required so the KB's retrieval layer
|
||||
// doesn't silently drift when we add fields to ScenarioSpec.
|
||||
|
||||
test("computeSignature is stable across reorderings of unrelated fields", () => {
|
||||
const a = {
|
||||
client: "Acme Corp",
|
||||
events: [
|
||||
{ kind: "baseline_fill", role: "Welder", count: 3, city: "Toledo", state: "OH" },
|
||||
],
|
||||
};
|
||||
const b = { ...a, date: "2026-05-01", contract: { deadline: "2026-05-15" } } as any;
|
||||
const c = { ...a, staffer: { id: "S-1", name: "X", tenure_months: 10, role: "senior" } } as any;
|
||||
const sigA = computeSignature(a);
|
||||
const sigB = computeSignature(b);
|
||||
const sigC = computeSignature(c);
|
||||
expect(sigA).toBe(sigB);
|
||||
expect(sigA).toBe(sigC);
|
||||
});
|
||||
|
||||
test("computeSignature changes when role changes", () => {
|
||||
const base = { client: "Acme", events: [{ kind: "baseline_fill", role: "Welder", count: 3, city: "Toledo", state: "OH" }] };
|
||||
const swapped = { client: "Acme", events: [{ kind: "baseline_fill", role: "Electrician", count: 3, city: "Toledo", state: "OH" }] };
|
||||
expect(computeSignature(base)).not.toBe(computeSignature(swapped));
|
||||
});
|
||||
|
||||
test("computeSignature changes when city or count changes", () => {
|
||||
const base = { client: "A", events: [{ kind: "baseline_fill", role: "Welder", count: 3, city: "Toledo", state: "OH" }] };
|
||||
const cityChange = { ...base, events: [{ ...base.events[0], city: "Detroit", state: "MI" }] };
|
||||
const countChange = { ...base, events: [{ ...base.events[0], count: 5 }] };
|
||||
expect(computeSignature(base)).not.toBe(computeSignature(cityChange));
|
||||
expect(computeSignature(base)).not.toBe(computeSignature(countChange));
|
||||
});
|
||||
|
||||
test("specDigest includes each event's role + city", () => {
|
||||
const spec = {
|
||||
client: "Acme",
|
||||
events: [
|
||||
{ kind: "baseline_fill", role: "Welder", count: 3, city: "Toledo", state: "OH" },
|
||||
{ kind: "emergency", role: "Loader", count: 2, city: "Chicago", state: "IL" },
|
||||
],
|
||||
};
|
||||
const digest = specDigest(spec);
|
||||
expect(digest).toContain("Acme");
|
||||
expect(digest).toContain("Welder");
|
||||
expect(digest).toContain("Toledo,OH");
|
||||
expect(digest).toContain("Loader");
|
||||
expect(digest).toContain("Chicago,IL");
|
||||
});
|
||||
@@ -35,7 +35,8 @@ import {
|
||||
reviewerPrompt,
|
||||
GATEWAY,
|
||||
} from "./agent.ts";
|
||||
import { indexRun, recommendFor, loadRecommendation, type PathwayRecommendation } from "./kb.ts";
|
||||
import { indexRun, recommendFor, loadRecommendation, snapshotConfig, type PathwayRecommendation } from "./kb.ts";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile, appendFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -1450,6 +1451,35 @@ async function main() {
|
||||
// per run. If no staffer or no tool_level, defaults hold.
|
||||
applyToolLevel(spec.staffer?.tool_level);
|
||||
|
||||
// Phase 22 — record the config snapshot each run gets. Lets the
|
||||
// error_corrections detector diff configs between fail→succeed pairs
|
||||
// and gives the KB a receipt of what was active for any given
|
||||
// outcome. Hash computed over the active model set + tool_level so
|
||||
// the same staffer running back-to-back with no config change
|
||||
// doesn't clutter the file.
|
||||
try {
|
||||
const activeModels = {
|
||||
executor: ACTIVE_EXECUTOR,
|
||||
reviewer: ACTIVE_REVIEWER,
|
||||
overview: OVERVIEW_MODEL,
|
||||
executor_cloud: String(ACTIVE_EXECUTOR_CLOUD),
|
||||
overview_cloud: String(ACTIVE_OVERVIEW_CLOUD),
|
||||
t3_disabled: String(ACTIVE_T3_DISABLED),
|
||||
tool_level: spec.staffer?.tool_level ?? "default",
|
||||
};
|
||||
const configHash = createHash("sha256")
|
||||
.update(JSON.stringify(activeModels))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
await snapshotConfig(
|
||||
configHash,
|
||||
activeModels,
|
||||
`scenario_start ${spec.client} ${spec.date} staffer=${spec.staffer?.id ?? "none"}`,
|
||||
);
|
||||
} catch (e) {
|
||||
console.log(` (config snapshot skipped: ${(e as Error).message})`);
|
||||
}
|
||||
|
||||
console.log(`▶ scenario: ${spec.client}, ${spec.date}, ${spec.events.length} events`);
|
||||
if (spec.staffer) {
|
||||
const level = spec.staffer.tool_level ?? "(default)";
|
||||
|
||||
Reference in New Issue
Block a user