checkpoint: matrix-agent-validated (2026-04-25)
Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.
WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.
WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
* UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
* REVISE: chains versions, parent.superseded_at + superseded_by stamped
* RETIRE: marks specific trace retired with reason, excluded from retrieval
* HISTORY: walks chain root→tip, cycle-safe
KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces
Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
profit
+11
@@ -0,0 +1,11 @@
|
||||
# Heavy test data — see REPLICATION.md for how to regenerate
|
||||
data/datasets/*.parquet
|
||||
data/vectors/*.parquet
|
||||
data/lance/
|
||||
data/_*/
|
||||
crates/ui/data/datasets/*.parquet
|
||||
target/
|
||||
node_modules/
|
||||
*.log
|
||||
.env
|
||||
/etc/lakehouse/secrets.toml
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"lakehouse": {
|
||||
"command": "bun",
|
||||
"args": ["run", "/home/profit/lakehouse/mcp-server/index.ts"],
|
||||
"env": {
|
||||
"LAKEHOUSE_URL": "http://localhost:3100",
|
||||
"MCP_TRANSPORT": "stdio"
|
||||
}
|
||||
},
|
||||
"gitea": {
|
||||
"command": "bunx",
|
||||
"args": ["gitea-mcp"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://git.agentview.dev",
|
||||
"GITEA_ACCESS_TOKEN": "SET_ME_FROM_GITEA_UI_USER_SETTINGS_APPLICATIONS"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+9830
File diff suppressed because it is too large
Load Diff
+53
@@ -0,0 +1,53 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/shared",
|
||||
"crates/proto",
|
||||
"crates/storaged",
|
||||
"crates/catalogd",
|
||||
"crates/queryd",
|
||||
"crates/aibridge",
|
||||
"crates/ingestd",
|
||||
"crates/vectord",
|
||||
"crates/journald",
|
||||
"crates/gateway",
|
||||
"crates/ui",
|
||||
"crates/lance-bench",
|
||||
"crates/vectord-lance",
|
||||
"crates/truth",
|
||||
"crates/validator",
|
||||
]
|
||||
|
||||
[workspace.dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
axum = "0.8"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||
thiserror = "2"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
tower-http = { version = "0.6", features = ["cors", "trace"] }
|
||||
object_store = { version = "0.12", features = ["aws"] }
|
||||
arrow = "55"
|
||||
parquet = { version = "55", features = ["arrow", "async"] }
|
||||
datafusion = "47"
|
||||
bytes = "1"
|
||||
futures = "0.3"
|
||||
sha2 = "0.10"
|
||||
url = "2"
|
||||
tonic = "0.13"
|
||||
prost = "0.13"
|
||||
tonic-build = "0.13"
|
||||
opentelemetry = "0.28"
|
||||
opentelemetry_sdk = { version = "0.28", features = ["rt-tokio"] }
|
||||
opentelemetry-stdout = { version = "0.28", features = ["trace"] }
|
||||
tracing-opentelemetry = "0.29"
|
||||
toml = "0.8"
|
||||
csv = "1"
|
||||
lopdf = "0.35"
|
||||
encoding_rs = "0.8"
|
||||
instant-distance = "0.6"
|
||||
tokio-postgres = { version = "0.7", features = ["with-serde_json-1", "with-chrono-0_4", "with-uuid-1"] }
|
||||
mysql_async = { version = "0.34", default-features = false, features = ["minimal"] }
|
||||
@@ -0,0 +1,60 @@
|
||||
# Lakehouse Claim Auditor
|
||||
|
||||
A Bun sub-agent that watches open PRs on Gitea, reads the ship-claims
|
||||
in commit messages and PR bodies, and **hard-blocks** merges when the
|
||||
code doesn't back the claim.
|
||||
|
||||
Rationale: when "compiles + one curl works" gets called "phase shipped,"
|
||||
placeholder code accumulates. This auditor runs every 90s, fetches
|
||||
each open PR, and subjects it to four checks:
|
||||
|
||||
1. **Static diff** — grep/parse looking for placeholder patterns
|
||||
2. **Dynamic** — runs the never-before-executed hybrid test fixture
|
||||
3. **Cloud inference** — asks `gpt-oss:120b` via `/v1/chat` to
|
||||
identify gaps in the diff
|
||||
4. **KB query** — looks up `data/_kb/` + observer for prior failure
|
||||
patterns on similar claims
|
||||
|
||||
Verdict is assembled, posted to Gitea as:
|
||||
- A **failing commit status** (hard block — branch protection
|
||||
prevents merge)
|
||||
- A **review comment** explaining every finding
|
||||
|
||||
## Run manually
|
||||
|
||||
```bash
|
||||
cd /home/profit/lakehouse
|
||||
bun run auditor/index.ts
|
||||
```
|
||||
|
||||
Defaults: polls every 90s, stops on `auditor.paused` file present.
|
||||
|
||||
## State
|
||||
|
||||
- `data/_auditor/state.json` — last-audited head SHA per PR
|
||||
- `data/_auditor/verdicts/{pr}-{sha}.json` — per-run verdict record
|
||||
- `data/_kb/audit_lessons.jsonl` — one row per block/warn finding,
|
||||
path-agnostic signature for dedup. Tailed by kb_query on each audit
|
||||
to surface recurring patterns (2+ distinct PRs with same signature
|
||||
→ info, 3-4 → warn, 5+ → block). This is how the auditor learns.
|
||||
- `data/_kb/scrum_reviews.jsonl` — scrum-master per-file reviews. If
|
||||
a file in the current PR has been scrum-reviewed, kb_query surfaces
|
||||
the review as a finding with the accepted model and attempt count.
|
||||
|
||||
## Where YOU edit
|
||||
|
||||
`auditor/policy.ts` — the verdict assembler. Controls which findings
|
||||
block vs warn vs inform. All other code is mechanical: fetching,
|
||||
running checks, posting to Gitea.
|
||||
|
||||
## Hard-block mechanism
|
||||
|
||||
1. Commit status is posted as `failure` with context `lakehouse/auditor`
|
||||
2. If `main` branch protection requires `lakehouse/auditor` status
|
||||
to pass, Gitea prevents merge
|
||||
3. When code is fixed and re-audit passes, status flips to `success`,
|
||||
merge unblocks
|
||||
|
||||
Enable branch protection (one-time, via Gitea UI or API):
|
||||
- `POST /repos/profit/lakehouse/branch_protections`
|
||||
- `{"branch_name": "main", "required_status_checks": {"contexts": ["lakehouse/auditor"]}}`
|
||||
@@ -0,0 +1,222 @@
|
||||
// Orchestrator — runs all four checks on a PR, assembles a verdict,
|
||||
// posts to Gitea. This is task #8's integration layer; the poller
|
||||
// (task #9) calls this once per PR on every fresh head SHA.
|
||||
//
|
||||
// Hard-block mechanism: commit status posted with state="failure"
|
||||
// and context="lakehouse/auditor". If `main` branch protection
|
||||
// requires that context to pass, merge is physically impossible
|
||||
// until the auditor re-audits a fixed commit and flips the status
|
||||
// to "success".
|
||||
//
|
||||
// Human-readable reasoning: posted as a PR issue comment (not a
|
||||
// review — reviews have self-review restrictions on Gitea and the
|
||||
// auditor currently uses the same PAT as the PR author).
|
||||
|
||||
import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import type { PrSnapshot, Verdict, Finding } from "./types.ts";
|
||||
import { getPrDiff, postCommitStatus, postIssueComment } from "./gitea.ts";
|
||||
import { parseClaims } from "./claim_parser.ts";
|
||||
import { assembleVerdict } from "./policy.ts";
|
||||
import { runStaticCheck } from "./checks/static.ts";
|
||||
import { runDynamicCheck } from "./checks/dynamic.ts";
|
||||
import { runInferenceCheck } from "./checks/inference.ts";
|
||||
import { runKbCheck } from "./checks/kb_query.ts";
|
||||
|
||||
const VERDICTS_DIR = "/home/profit/lakehouse/data/_auditor/verdicts";
|
||||
// Playbook for audit findings — one row per block/warn finding from a
|
||||
// verdict. kb_query tails this next audit and escalates recurrences.
|
||||
// Structured as JSONL so it's cheap to append and cheap to tail.
|
||||
const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
|
||||
|
||||
export interface AuditOptions {
|
||||
// Skip the cloud inference call (fast path for iteration). Default false.
|
||||
skip_inference?: boolean;
|
||||
// Skip the dynamic check (avoid running the hybrid fixture every PR,
|
||||
// since it hits live services and mutates playbook state). Default false
|
||||
// on `main`-branch-target PRs, true when auditing feature branches
|
||||
// where the fixture would pollute state. Caller decides.
|
||||
skip_dynamic?: boolean;
|
||||
// Skip Gitea posting — useful for dry-runs / local testing.
|
||||
// Default false.
|
||||
dry_run?: boolean;
|
||||
}
|
||||
|
||||
export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<Verdict> {
|
||||
const t0 = Date.now();
|
||||
const diff = await getPrDiff(pr.number);
|
||||
const { claims } = parseClaims(pr);
|
||||
|
||||
// Run checks in parallel where they don't share mutable state.
|
||||
// Static + kb_query + inference are all read-only. Dynamic mutates
|
||||
// playbook state (nonce-scoped per run, but still live) so if
|
||||
// skip_dynamic is false we still run it in parallel — the mutation
|
||||
// is namespaced.
|
||||
const [staticFindings, dynamicFindings, inferenceFindings, kbFindings] = await Promise.all([
|
||||
runStaticCheck(diff),
|
||||
opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
|
||||
opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff, { pr_number: pr.number, head_sha: pr.head_sha }),
|
||||
runKbCheck(claims, pr.files.map(f => f.path)),
|
||||
]);
|
||||
|
||||
const allFindings: Finding[] = [
|
||||
...staticFindings,
|
||||
...dynamicFindings,
|
||||
...inferenceFindings,
|
||||
...kbFindings,
|
||||
];
|
||||
|
||||
const duration_ms = Date.now() - t0;
|
||||
const metrics = {
|
||||
audit_duration_ms: duration_ms,
|
||||
findings_total: allFindings.length,
|
||||
findings_block: allFindings.filter(f => f.severity === "block").length,
|
||||
findings_warn: allFindings.filter(f => f.severity === "warn").length,
|
||||
findings_info: allFindings.filter(f => f.severity === "info").length,
|
||||
claims_strong: claims.filter(c => c.strength === "strong").length,
|
||||
claims_moderate: claims.filter(c => c.strength === "moderate").length,
|
||||
claims_weak: claims.filter(c => c.strength === "weak").length,
|
||||
claims_empirical: claims.filter(c => c.strength === "empirical").length,
|
||||
claims_total: claims.length,
|
||||
diff_bytes: diff.length,
|
||||
};
|
||||
|
||||
const verdict = assembleVerdict(allFindings, metrics, pr.number, pr.head_sha);
|
||||
|
||||
await persistVerdict(verdict);
|
||||
|
||||
// Feedback loop — every block/warn finding becomes a row in
|
||||
// audit_lessons.jsonl, dedup-keyed by (check, normalized-summary).
|
||||
// The next audit's kb_query reads these and escalates recurring
|
||||
// findings so we don't lose the "this pattern has been flagged
|
||||
// before" signal across runs. Fire-and-forget; failure here must
|
||||
// not break the audit.
|
||||
appendAuditLessons(verdict).catch(e =>
|
||||
console.error(`[audit] audit_lessons append failed: ${(e as Error).message}`));
|
||||
|
||||
if (!opts.dry_run) {
|
||||
await postToGitea(verdict);
|
||||
}
|
||||
|
||||
return verdict;
|
||||
}
|
||||
|
||||
// Normalizes a finding summary for dedup: strips path-specific tails
|
||||
// ("in path/to/file.ts" → "in <file>"), line numbers, and long
|
||||
// commit-hash snippets. The goal is: the SAME class of finding on
|
||||
// DIFFERENT files should share a signature, so we can measure
|
||||
// "this pattern keeps showing up."
|
||||
function normalizedSignature(f: Finding): string {
|
||||
const summary = String(f.summary)
|
||||
.replace(/\bin\s+\S+\.(ts|rs|js|py|md)\b/gi, "in <file>")
|
||||
.replace(/:\+?\d+\b/g, ":<line>")
|
||||
.replace(/[0-9a-f]{8,}/gi, "<hash>")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 240);
|
||||
const src = `${f.check}::${f.severity}::${summary}`;
|
||||
return createHash("sha256").update(src).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
async function appendAuditLessons(v: Verdict): Promise<void> {
|
||||
const actionable = v.findings.filter(f => f.severity === "block" || f.severity === "warn");
|
||||
if (actionable.length === 0) return;
|
||||
await mkdir(join(AUDIT_LESSONS_JSONL, ".."), { recursive: true });
|
||||
const rows: string[] = [];
|
||||
for (const f of actionable) {
|
||||
rows.push(JSON.stringify({
|
||||
signature: normalizedSignature(f),
|
||||
check: f.check,
|
||||
severity: f.severity,
|
||||
summary: f.summary,
|
||||
pr_number: v.pr_number,
|
||||
head_sha: v.head_sha,
|
||||
audited_at: v.audited_at,
|
||||
}));
|
||||
}
|
||||
await appendFile(AUDIT_LESSONS_JSONL, rows.join("\n") + "\n");
|
||||
}
|
||||
|
||||
async function persistVerdict(v: Verdict): Promise<void> {
|
||||
await mkdir(VERDICTS_DIR, { recursive: true });
|
||||
const filename = `${v.pr_number}-${v.head_sha.slice(0, 12)}.json`;
|
||||
await writeFile(join(VERDICTS_DIR, filename), JSON.stringify(v, null, 2));
|
||||
}
|
||||
|
||||
export async function postToGitea(v: Verdict): Promise<void> {
|
||||
// 1. Commit status — the hard block signal (if branch protection
|
||||
// is configured to require lakehouse/auditor on main).
|
||||
const state = v.overall === "approve" ? "success" : "failure";
|
||||
await postCommitStatus({
|
||||
sha: v.head_sha,
|
||||
state,
|
||||
context: "lakehouse/auditor",
|
||||
description: v.one_liner,
|
||||
target_url: "", // no URL yet; could point to a verdicts dashboard
|
||||
});
|
||||
|
||||
// 2. Issue comment — the reasoning. Gated so we don't spam the PR
|
||||
// with identical comments on re-audits of the same SHA. Caller
|
||||
// (poller) ensures we only re-audit fresh SHAs, but a dedup
|
||||
// marker inside the body keeps it idempotent if re-run.
|
||||
const body = formatReviewBody(v);
|
||||
await postIssueComment({ pr_number: v.pr_number, body });
|
||||
}
|
||||
|
||||
function formatReviewBody(v: Verdict): string {
|
||||
const byCheck: Record<string, Finding[]> = {};
|
||||
for (const f of v.findings) {
|
||||
(byCheck[f.check] ||= []).push(f);
|
||||
}
|
||||
|
||||
const verdictEmoji =
|
||||
v.overall === "approve" ? "✅" :
|
||||
v.overall === "request_changes" ? "⚠️" :
|
||||
"🛑";
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(`## Auditor verdict: ${verdictEmoji} \`${v.overall}\``);
|
||||
lines.push("");
|
||||
lines.push(`**One-liner:** ${v.one_liner}`);
|
||||
lines.push(`**Head SHA:** \`${v.head_sha.slice(0, 12)}\``);
|
||||
lines.push(`**Audited at:** ${v.audited_at}`);
|
||||
lines.push("");
|
||||
|
||||
// Per-check sections, only if the check produced findings.
|
||||
const checkOrder = ["static", "dynamic", "inference", "kb_query"] as const;
|
||||
for (const check of checkOrder) {
|
||||
const fs = byCheck[check] ?? [];
|
||||
if (fs.length === 0) continue;
|
||||
const bySev = {
|
||||
block: fs.filter(f => f.severity === "block").length,
|
||||
warn: fs.filter(f => f.severity === "warn").length,
|
||||
info: fs.filter(f => f.severity === "info").length,
|
||||
};
|
||||
lines.push(`<details><summary><b>${check}</b> — ${fs.length} findings (${bySev.block} block, ${bySev.warn} warn, ${bySev.info} info)</summary>`);
|
||||
lines.push("");
|
||||
for (const f of fs) {
|
||||
const mark = f.severity === "block" ? "🛑" : f.severity === "warn" ? "⚠️" : "ℹ️";
|
||||
lines.push(`${mark} **${f.severity}** — ${f.summary}`);
|
||||
for (const e of f.evidence.slice(0, 3)) {
|
||||
lines.push(` - \`${e.slice(0, 180).replace(/\n/g, " ")}\``);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("</details>");
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
lines.push("### Metrics");
|
||||
lines.push("```json");
|
||||
lines.push(JSON.stringify(v.metrics, null, 2));
|
||||
lines.push("```");
|
||||
lines.push("");
|
||||
lines.push(`<sub>Lakehouse auditor · SHA ${v.head_sha.slice(0, 8)} · re-audit on new commit flips the status automatically.</sub>`);
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function stubFinding(check: "dynamic" | "inference", why: string): Finding[] {
|
||||
return [{ check, severity: "info", summary: `${check} check skipped — ${why}`, evidence: [why] }];
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// One-shot dry-run audit of a single PR. Useful for verifying check
|
||||
// behavior (kb_query scrum surfacing, inference prompts, etc.) without
|
||||
// posting to Gitea. Does NOT touch state.json and does NOT post
|
||||
// commit status or PR comments.
|
||||
//
|
||||
// Run: bun run auditor/audit_one.ts <pr-number>
|
||||
|
||||
import { getPrSnapshot } from "./gitea.ts";
|
||||
import { auditPr } from "./audit.ts";
|
||||
|
||||
async function main() {
|
||||
const prNumRaw = process.argv[2];
|
||||
if (!prNumRaw) {
|
||||
console.error("usage: bun run auditor/audit_one.ts <pr-number>");
|
||||
process.exit(2);
|
||||
}
|
||||
const prNum = Number(prNumRaw);
|
||||
if (!Number.isFinite(prNum)) {
|
||||
console.error(`invalid PR number: ${prNumRaw}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
console.log(`[audit_one] fetching PR #${prNum}...`);
|
||||
const pr = await getPrSnapshot(prNum);
|
||||
console.log(`[audit_one] PR #${pr.number}: "${pr.title}" (head=${pr.head_sha.slice(0, 12)})`);
|
||||
console.log(`[audit_one] files in diff: ${pr.files.length}`);
|
||||
for (const f of pr.files) console.log(` - ${f.path} (+${f.additions}/-${f.deletions})`);
|
||||
console.log("");
|
||||
|
||||
const verdict = await auditPr(pr, {
|
||||
dry_run: true, // no Gitea posting
|
||||
skip_dynamic: true, // don't run fixture
|
||||
skip_inference: process.env.LH_AUDITOR_SKIP_INFERENCE === "1",
|
||||
});
|
||||
|
||||
console.log("\n═══ VERDICT ═══");
|
||||
console.log(`overall: ${verdict.overall}`);
|
||||
console.log(`one-liner: ${verdict.one_liner}`);
|
||||
console.log(`findings: total=${verdict.metrics.findings_total} block=${verdict.metrics.findings_block} warn=${verdict.metrics.findings_warn} info=${verdict.metrics.findings_info}`);
|
||||
console.log("");
|
||||
|
||||
// Print findings, highlighting kb_query scrum surfacing
|
||||
const byCheck: Record<string, typeof verdict.findings> = {};
|
||||
for (const f of verdict.findings) (byCheck[f.check] ||= []).push(f);
|
||||
|
||||
for (const [check, findings] of Object.entries(byCheck)) {
|
||||
console.log(`── ${check} (${findings.length}) ──`);
|
||||
for (const f of findings) {
|
||||
const tag = f.severity === "block" ? "🛑" : f.severity === "warn" ? "⚠️ " : "ℹ️ ";
|
||||
console.log(` ${tag} [${f.severity}] ${f.summary}`);
|
||||
if (f.summary.includes("scrum-master")) {
|
||||
for (const e of f.evidence) {
|
||||
console.log(` → ${e.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const scrumFindings = verdict.findings.filter(f => f.summary.includes("scrum-master"));
|
||||
console.log("");
|
||||
console.log(`═══ SCRUM WIRE CHECK: ${scrumFindings.length} scrum-master findings surfaced by kb_query ═══`);
|
||||
if (scrumFindings.length === 0) {
|
||||
console.log(" (none — either no matching scrum_reviews.jsonl rows, or files didn't match PR diff)");
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error("[audit_one] fatal:", e); process.exit(1); });
|
||||
@@ -0,0 +1,91 @@
|
||||
// Dynamic execution check — runs the hybrid fixture and maps its
|
||||
// layer results to auditor Findings.
|
||||
//
|
||||
// A layer that fails with a "not implemented / 404 / slice N" error
|
||||
// gets severity=info (honest placeholder signal). A layer that fails
|
||||
// any other way gets severity=warn (something actually broke).
|
||||
// An info-level summary finding is always emitted carrying the real
|
||||
// numbers — shipped/placeholder phase counts, per-layer latency.
|
||||
|
||||
import { runHybridFixture } from "../fixtures/hybrid_38_40_45.ts";
|
||||
import type { Finding } from "../types.ts";
|
||||
|
||||
const PLACEHOLDER_MARKERS = [
|
||||
"unimplemented",
|
||||
" 404 ", "(404)", " 405 ", "(405)",
|
||||
"slice 3", "slice 4", "slice 5",
|
||||
"endpoint not built", "not yet",
|
||||
];
|
||||
|
||||
function isPlaceholderFailure(err?: string): boolean {
|
||||
if (!err) return false;
|
||||
const low = err.toLowerCase();
|
||||
return PLACEHOLDER_MARKERS.some(m => low.includes(m.toLowerCase()));
|
||||
}
|
||||
|
||||
export async function runDynamicCheck(): Promise<Finding[]> {
|
||||
const findings: Finding[] = [];
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await runHybridFixture();
|
||||
} catch (e) {
|
||||
// Fixture itself crashed — can't run dynamic check at all.
|
||||
return [
|
||||
{
|
||||
check: "dynamic",
|
||||
severity: "warn",
|
||||
summary: `hybrid fixture crashed before completing: ${(e as Error).message.slice(0, 140)}`,
|
||||
evidence: [(e as Error).message],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// Per-layer findings for every non-ok layer.
|
||||
for (const layer of result.layers) {
|
||||
if (layer.ok) continue;
|
||||
const placeholder = isPlaceholderFailure(layer.error);
|
||||
findings.push({
|
||||
check: "dynamic",
|
||||
severity: placeholder ? "info" : "warn",
|
||||
summary: placeholder
|
||||
? `hybrid fixture layer ${layer.layer} (Phase ${layer.phase}) honestly reports unimplemented`
|
||||
: `hybrid fixture layer ${layer.layer} (Phase ${layer.phase}) failed — not a placeholder, a real failure`,
|
||||
evidence: [
|
||||
`evidence: ${layer.evidence.slice(0, 160)}`,
|
||||
...(layer.error ? [`error: ${layer.error.slice(0, 160)}`] : []),
|
||||
`latency_ms: ${layer.latency_ms}`,
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
// One overall summary with real numbers so the report shows what
|
||||
// DID pass plus per-layer timing.
|
||||
const metrics_preview = Object.entries(result.real_numbers)
|
||||
.slice(0, 10)
|
||||
.map(([k, v]) => `${k}=${v}`);
|
||||
findings.push({
|
||||
check: "dynamic",
|
||||
severity: "info",
|
||||
summary: `hybrid fixture overall=${result.overall}, shipped [${result.shipped_phases.join(", ")}], placeholder [${result.placeholder_phases.join(", ")}]`,
|
||||
evidence: metrics_preview.length > 0 ? metrics_preview : ["no metrics emitted"],
|
||||
});
|
||||
|
||||
// If the fixture ran at all but nothing passed, elevate one of the
|
||||
// summary findings to warn — something more than "all honest
|
||||
// placeholders" is wrong.
|
||||
if (result.overall === "fail") {
|
||||
findings.push({
|
||||
check: "dynamic",
|
||||
severity: "warn",
|
||||
summary: `hybrid fixture: 0 layers passed (overall=fail)`,
|
||||
evidence: [
|
||||
"a total fixture fail usually means a precondition service is down",
|
||||
"(gateway /health / sidecar / Langfuse /v1/chat) — NOT necessarily",
|
||||
"the PR's code problem. Check service status before blaming the PR.",
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
@@ -0,0 +1,666 @@
|
||||
// Cloud inference check — wraps the proven run_codereview pattern
|
||||
// from llm_team_ui.py (same 3-stage framing, same cloud model) to
|
||||
// critique a PR's claims against its diff.
|
||||
//
|
||||
// Proved out 2026-04-22 via /tmp/codereview_runner.py — gpt-oss:120b
|
||||
// caught a real ternary bug in auditor/fixtures/hybrid_38_40_45.ts
|
||||
// that unit tests missed. This module reuses the reviewer prompt
|
||||
// shape (bugs / security / performance / style / edge cases) and
|
||||
// adds claim-vs-diff specific framing.
|
||||
//
|
||||
// Call surface: runInferenceCheck(claims, diff) → Finding[].
|
||||
// Cloud latency budget: ~60s (gpt-oss:120b reviewer typically 35-50s
|
||||
// with a 15KB diff + claim list).
|
||||
|
||||
import type { Claim, Finding } from "../types.ts";
|
||||
import { Glob } from "bun";
|
||||
import { readFile, mkdir, appendFile } from "node:fs/promises";
|
||||
import { extractFacts } from "../fact_extractor.ts";
|
||||
|
||||
const GATEWAY = process.env.LH_GATEWAY_URL ?? "http://localhost:3100";
|
||||
const MODEL = process.env.LH_AUDITOR_REVIEW_MODEL ?? "gpt-oss:120b";
|
||||
// Tie-breaker for claims where the N=3 consensus produces a 1-1-1
|
||||
// split (genuinely borderline). Different architecture from the
|
||||
// primary reviewer (gpt-oss) so the tie-break isn't correlated with
|
||||
// the original disagreement. qwen3-coder:480b is a newer coding
|
||||
// specialist at 480B params, well-suited to PR-diff claim verification
|
||||
// and distinct in training lineage from gpt-oss.
|
||||
const TIEBREAKER_MODEL = process.env.LH_AUDITOR_TIEBREAKER_MODEL ?? "qwen3-coder:480b";
|
||||
const N_CONSENSUS = Number(process.env.LH_AUDITOR_CONSENSUS_N ?? 3);
|
||||
const AUDIT_DISCREPANCIES_JSONL = "/home/profit/lakehouse/data/_kb/audit_discrepancies.jsonl";
|
||||
// 40KB comfortably fits gpt-oss:120b's context. PR #1 (~39KB) was
|
||||
// previously truncated at 15KB causing the reviewer to miss later
|
||||
// files (gitea.ts, policy.ts) and flag "no Gitea client present" as a
|
||||
// block finding when the file was simply outside the truncation window.
|
||||
//
|
||||
// Above this threshold we curate via tree-split rather than truncate,
|
||||
// following the scrum_master pattern: shard the diff, summarize each
|
||||
// shard against the claim-verification task, merge into a compact
|
||||
// scratchpad, then ask the cloud to verify claims against the
|
||||
// scratchpad. This gives the cloud full-PR fidelity without bursting
|
||||
// its context window (observed failure mode: empty response or
|
||||
// unparseable output when prompt exceeds model's comfortable range).
|
||||
const MAX_DIFF_CHARS = 40000;
|
||||
// Tree-split kicks in above this. 30KB is below MAX_DIFF_CHARS so we
|
||||
// curate BEFORE truncation would happen — never lose signal to a hard
|
||||
// cut. Shard size is chosen so ~10 shards cover PR #8-size diffs in a
|
||||
// reasonable round-trip budget.
|
||||
const CURATION_THRESHOLD = 30000;
|
||||
const DIFF_SHARD_SIZE = 4500;
|
||||
const CALL_TIMEOUT_MS = 120_000;
|
||||
const REPO_ROOT = "/home/profit/lakehouse";
|
||||
|
||||
export interface InferenceContext {
|
||||
pr_number: number;
|
||||
head_sha: string;
|
||||
}
|
||||
|
||||
const AUDIT_FACTS_JSONL = "/home/profit/lakehouse/data/_kb/audit_facts.jsonl";
|
||||
|
||||
export async function runInferenceCheck(
|
||||
claims: Claim[],
|
||||
diff: string,
|
||||
ctx?: InferenceContext,
|
||||
): Promise<Finding[]> {
|
||||
if (claims.length === 0) {
|
||||
return [{
|
||||
check: "inference",
|
||||
severity: "info",
|
||||
summary: "no ship-claims extracted — skipping cloud inference",
|
||||
evidence: ["parser returned empty claim list; nothing to verify against cloud"],
|
||||
}];
|
||||
}
|
||||
|
||||
// Empirical claims (runtime metrics / observed outcomes) can't be
|
||||
// verified from the diff. Drop them from the cloud prompt so the
|
||||
// reviewer doesn't chase ghosts. A future `runtime_evidence` check
|
||||
// can validate these against data/_kb/*/summary.json outputs.
|
||||
const verifiable = claims.filter(c => c.strength !== "empirical");
|
||||
const empiricalCount = claims.length - verifiable.length;
|
||||
if (verifiable.length === 0) {
|
||||
return [{
|
||||
check: "inference",
|
||||
severity: "info",
|
||||
summary: `all ${claims.length} claims are empirical (runtime metrics) — skipping cloud inference`,
|
||||
evidence: [`empirical claims can't be verified from a static diff; needs runtime-evidence check`],
|
||||
}];
|
||||
}
|
||||
|
||||
// Diff source for the cloud prompt — either the raw diff (small
|
||||
// enough to fit), or a tree-split scratchpad (curation layer). We
|
||||
// prefer curation to truncation: truncation silently drops files
|
||||
// past the window; curation summarizes them so the cloud still sees
|
||||
// what changed, just densified.
|
||||
let diffForPrompt: string;
|
||||
let curationNote = "";
|
||||
if (diff.length > CURATION_THRESHOLD) {
|
||||
const ts = await treeSplitDiff(diff, verifiable);
|
||||
diffForPrompt = ts.scratchpad;
|
||||
curationNote = ` (curated: ${diff.length} chars → ${ts.shards} shards → scratchpad ${ts.scratchpad.length} chars)`;
|
||||
} else {
|
||||
diffForPrompt = diff;
|
||||
}
|
||||
// Belt-and-suspenders truncation — even a tree-split scratchpad
|
||||
// shouldn't exceed MAX_DIFF_CHARS in practice, but guard anyway so
|
||||
// pathological inputs can't burst the prompt.
|
||||
const truncated = diffForPrompt.length > MAX_DIFF_CHARS
|
||||
? diffForPrompt.slice(0, MAX_DIFF_CHARS) + `\n...[${diffForPrompt.length - MAX_DIFF_CHARS} more chars truncated]`
|
||||
: diffForPrompt;
|
||||
|
||||
// Build the reviewer prompt in the same shape as run_codereview's
|
||||
// review stage (llm_team_ui.py:10950), adapted for claim verification:
|
||||
// "Task: ..."
|
||||
// "Code: ..."
|
||||
// "Review: bugs/security/perf/style/edge. Provide corrected code."
|
||||
// We add: claim list upfront + ask for structured JSON verdict.
|
||||
//
|
||||
// When the diff was curated (tree-split scratchpad), we add an
|
||||
// explicit anti-false-positive instruction: the scratchpad is a
|
||||
// distillation, not the full source, so absence-from-scratchpad is
|
||||
// NOT evidence of absence-from-diff. Mirrors the fix we made in
|
||||
// scrum_master's review prompt for the same class of error.
|
||||
const isCurated = curationNote.length > 0;
|
||||
const curationGuard = isCurated
|
||||
? [
|
||||
"",
|
||||
"CRITICAL: the 'Diff' below is a curated multi-shard scratchpad,",
|
||||
"NOT the full raw diff. The scratchpad distills each shard down",
|
||||
"to facts useful for claim verification and drops the rest.",
|
||||
"DO NOT flag a function/field/feature as 'missing' or 'not",
|
||||
"implemented' based solely on its absence from the scratchpad —",
|
||||
"absence in a distillation is NOT evidence of absence in the",
|
||||
"actual diff. Only judge a claim NOT BACKED when the scratchpad",
|
||||
"DIRECTLY contradicts it (e.g. scratchpad shows the function was",
|
||||
"added empty, or shows the claimed code path is a stub).",
|
||||
"Skip the unflagged_gaps section entirely when operating on a",
|
||||
"curated scratchpad — you can't reliably detect gaps from a",
|
||||
"distillation, and false positives there are worse than misses.",
|
||||
].join("\n")
|
||||
: "";
|
||||
const systemMsg = [
|
||||
"You review pull-request diffs against the author's own ship-claims.",
|
||||
"For each claim, decide: is it backed by actual code in the diff, or is",
|
||||
"it placeholder / aspirational / unwired?",
|
||||
"",
|
||||
"A claim is BACKED when the diff contains a real code path that delivers",
|
||||
"the claimed behavior. A claim is NOT BACKED when:",
|
||||
" - the claim asserts functionality but the diff only adds types/fields",
|
||||
" with no consumer",
|
||||
" - the claim mentions tests but no test function was added",
|
||||
" - the claim claims integration but the integration point is a stub",
|
||||
" - the diff contains unimplemented!() / todo!() / TODO comments",
|
||||
" - the claim says 'works end-to-end' but the diff has no end-to-end test",
|
||||
curationGuard,
|
||||
"",
|
||||
"Respond with strict JSON only. No prose before or after. Shape:",
|
||||
"{",
|
||||
' "claim_verdicts": [',
|
||||
' {"claim_idx": 0, "backed": false, "evidence": "short reason"}',
|
||||
" ],",
|
||||
' "unflagged_gaps": [',
|
||||
' {"location": "file:line", "summary": "short description"}',
|
||||
" ]",
|
||||
"}",
|
||||
].join("\n");
|
||||
|
||||
const userMsg = [
|
||||
`Ship-claims the author made (numbered 0..N-1):`,
|
||||
verifiable.map((c, i) => ` ${i}. [${c.strength}] "${c.text}" at ${c.location}`).join("\n"),
|
||||
"",
|
||||
`Diff:`,
|
||||
"```",
|
||||
truncated,
|
||||
"```",
|
||||
"",
|
||||
`For each numbered claim above, emit a claim_verdicts entry. For gaps the`,
|
||||
`author DIDN'T claim but that look like placeholder code, emit unflagged_gaps.`,
|
||||
`Strict JSON only, matching the shape described. No prose outside JSON.`,
|
||||
].join("\n");
|
||||
|
||||
// N=3 consensus — run the primary reviewer in parallel, collect
|
||||
// all three parsed responses, majority-vote per claim. Parallel
|
||||
// (Promise.all) because each call is ~20-30s and they're independent;
|
||||
// wall-clock stays ~same as single call, cost 3x tokens. Empirical
|
||||
// justification: in 3-run determinism tests, 7/8 findings were
|
||||
// stable but 1 flipped across runs — majority vote stabilizes the
|
||||
// flipping class without losing the stable signal.
|
||||
const primaryRuns = await Promise.all(
|
||||
Array.from({ length: N_CONSENSUS }, () =>
|
||||
runCloudInference(systemMsg, userMsg, MODEL)),
|
||||
);
|
||||
|
||||
const parsedRuns = primaryRuns.filter(r => r.parsed !== null);
|
||||
if (parsedRuns.length === 0) {
|
||||
// All N calls failed. Surface the first-run diagnostic so the
|
||||
// operator sees *why* (unreachable / non-200 / unparseable).
|
||||
const first = primaryRuns[0];
|
||||
return [{
|
||||
check: "inference",
|
||||
severity: "info",
|
||||
summary: `cloud inference all ${N_CONSENSUS} consensus runs failed — ${first.error ?? "unknown"}`,
|
||||
evidence: [
|
||||
`first-run diagnostic: ${first.diagnostic ?? "(none)"}`,
|
||||
`successful runs: 0 / ${N_CONSENSUS}`,
|
||||
],
|
||||
}];
|
||||
}
|
||||
|
||||
// Aggregate votes per claim_idx.
|
||||
interface Votes { trues: number; falses: number; evidences: string[] }
|
||||
const votesByClaim = new Map<number, Votes>();
|
||||
const unflaggedByRun: any[][] = [];
|
||||
let totalTokens = 0;
|
||||
for (const run of parsedRuns) {
|
||||
totalTokens += run.tokens;
|
||||
unflaggedByRun.push(Array.isArray(run.parsed?.unflagged_gaps) ? run.parsed.unflagged_gaps : []);
|
||||
for (const v of run.parsed?.claim_verdicts ?? []) {
|
||||
const idx = Number(v?.claim_idx);
|
||||
if (!Number.isFinite(idx)) continue;
|
||||
const rec = votesByClaim.get(idx) ?? { trues: 0, falses: 0, evidences: [] };
|
||||
if (v.backed === false) {
|
||||
rec.falses++;
|
||||
rec.evidences.push(String(v.evidence ?? ""));
|
||||
} else if (v.backed === true) {
|
||||
rec.trues++;
|
||||
}
|
||||
votesByClaim.set(idx, rec);
|
||||
}
|
||||
}
|
||||
|
||||
const findings: Finding[] = [];
|
||||
|
||||
// Summary finding so the verdict layer knows the check ran.
|
||||
findings.push({
|
||||
check: "inference",
|
||||
severity: "info",
|
||||
summary: `cloud review completed (model=${MODEL}, consensus=${parsedRuns.length}/${N_CONSENSUS}, tokens=${totalTokens})${curationNote}`,
|
||||
evidence: [
|
||||
`claims voted: ${votesByClaim.size}`,
|
||||
`parsed runs: ${parsedRuns.length} / ${N_CONSENSUS}`,
|
||||
],
|
||||
});
|
||||
|
||||
// Per-claim majority vote; tie-break if no majority.
|
||||
const discrepancies: Array<{
|
||||
claim_idx: number;
|
||||
claim_text: string;
|
||||
votes: { trues: number; falses: number };
|
||||
resolution: "majority_backed" | "majority_not_backed" | "tiebreaker_backed" | "tiebreaker_not_backed" | "unresolved";
|
||||
tiebreaker_model?: string;
|
||||
}> = [];
|
||||
|
||||
for (const [idx, votes] of votesByClaim) {
|
||||
const claim = verifiable[idx];
|
||||
if (!claim) continue;
|
||||
const totalVotes = votes.trues + votes.falses;
|
||||
let notBacked: boolean | null = null;
|
||||
let resolution: typeof discrepancies[number]["resolution"] = "majority_backed";
|
||||
let evidenceText = "";
|
||||
let tbModel: string | undefined;
|
||||
|
||||
if (votes.falses > votes.trues) {
|
||||
notBacked = true;
|
||||
resolution = "majority_not_backed";
|
||||
evidenceText = votes.evidences[0] ?? "(no reason given)";
|
||||
} else if (votes.trues > votes.falses) {
|
||||
notBacked = false;
|
||||
resolution = "majority_backed";
|
||||
} else {
|
||||
// Tie. Run tie-breaker with a different-architecture model.
|
||||
const tb = await runCloudInference(systemMsg, userMsg, TIEBREAKER_MODEL);
|
||||
if (tb.parsed) {
|
||||
const tv = (tb.parsed.claim_verdicts ?? []).find((v: any) => Number(v?.claim_idx) === idx);
|
||||
if (tv?.backed === false) {
|
||||
notBacked = true;
|
||||
resolution = "tiebreaker_not_backed";
|
||||
evidenceText = `(tie-breaker ${TIEBREAKER_MODEL}) ${String(tv.evidence ?? "")}`;
|
||||
tbModel = TIEBREAKER_MODEL;
|
||||
} else if (tv?.backed === true) {
|
||||
notBacked = false;
|
||||
resolution = "tiebreaker_backed";
|
||||
tbModel = TIEBREAKER_MODEL;
|
||||
} else {
|
||||
resolution = "unresolved";
|
||||
}
|
||||
} else {
|
||||
resolution = "unresolved";
|
||||
}
|
||||
}
|
||||
|
||||
// Log every case where the N runs disagreed — discrepancies are
|
||||
// signal, not noise. Separate from audit_lessons.jsonl because
|
||||
// they're about the *auditor's* quality, not the PR's quality.
|
||||
const disagreed = totalVotes >= 2 && votes.trues > 0 && votes.falses > 0;
|
||||
if (disagreed || resolution.startsWith("tiebreaker") || resolution === "unresolved") {
|
||||
discrepancies.push({
|
||||
claim_idx: idx,
|
||||
claim_text: claim.text,
|
||||
votes: { trues: votes.trues, falses: votes.falses },
|
||||
resolution,
|
||||
tiebreaker_model: tbModel,
|
||||
});
|
||||
}
|
||||
|
||||
if (notBacked === true) {
|
||||
const sev: Finding["severity"] = claim.strength === "strong" ? "block"
|
||||
: claim.strength === "moderate" ? "warn"
|
||||
: "info";
|
||||
findings.push({
|
||||
check: "inference",
|
||||
severity: sev,
|
||||
claim_text: claim.text,
|
||||
summary: `cloud: claim not backed — "${claim.text.slice(0, 100)}"`,
|
||||
evidence: [
|
||||
`at ${claim.location}`,
|
||||
`consensus: ${votes.falses}/${totalVotes} not-backed (resolution: ${resolution})`,
|
||||
`cloud reason: ${evidenceText.slice(0, 200)}`,
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Persist discrepancies so we can measure consensus drift over time.
|
||||
if (discrepancies.length > 0 && ctx) {
|
||||
persistDiscrepancies(ctx, discrepancies).catch(e =>
|
||||
console.error(`[inference] discrepancy log failed: ${(e as Error).message}`));
|
||||
}
|
||||
|
||||
// Use first run's parsed for downstream unflagged_gaps processing.
|
||||
const parsed = parsedRuns[0].parsed;
|
||||
|
||||
// Route the curated scratchpad through llm_team's extract-facts
|
||||
// pipeline when we have (a) a curated scratchpad (best signal about
|
||||
// what the PR actually changed) and (b) PR context to scope facts.
|
||||
// AWAITED (not fire-and-forget) so CLI callers like audit_one.ts
|
||||
// don't exit before extraction lands; the systemd poller has plenty
|
||||
// of headroom (90s cycle vs ~15s extraction). A failure inside
|
||||
// extractAndPersistFacts is caught + logged but never throws.
|
||||
if (isCurated && ctx && process.env.LH_AUDITOR_SKIP_EXTRACT !== "1") {
|
||||
try {
|
||||
await extractAndPersistFacts(diffForPrompt, ctx);
|
||||
} catch (e) {
|
||||
console.error(`[inference] fact extraction failed: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Belt-and-suspenders: when operating on a curated scratchpad, drop
|
||||
// the unflagged_gaps section entirely. The distillation can't
|
||||
// reliably ground gap-detection, and false positives are worse than
|
||||
// misses for this signal class. The systemMsg already asks the
|
||||
// cloud to skip this section when curated — but the model may still
|
||||
// emit it, so we filter here too.
|
||||
const gapsToEmit = isCurated ? [] : (parsed.unflagged_gaps ?? []);
|
||||
for (const g of gapsToEmit) {
|
||||
const summary = String(g?.summary ?? "?");
|
||||
const location = String(g?.location ?? "?");
|
||||
// False-positive guard — when the cloud says "X not defined in this
|
||||
// diff" or "missing implementation of X", the cloud may just mean
|
||||
// "X is not in the added lines," not "X doesn't exist in the repo."
|
||||
// Extract candidate symbol names and grep the repo. If any symbol
|
||||
// is defined elsewhere, drop the finding — it's a known-symbol
|
||||
// reference, not a placeholder.
|
||||
if (/not\s+defined|missing\s+implementation|never\s+referenced\s+or\s+integrated/i.test(summary)) {
|
||||
const symbols = extractSymbols(summary);
|
||||
if (symbols.length > 0) {
|
||||
const resolved = await symbolsExistInRepo(symbols);
|
||||
if (resolved.length === symbols.length) {
|
||||
// Every named symbol exists somewhere in the repo — silent drop.
|
||||
continue;
|
||||
}
|
||||
if (resolved.length > 0) {
|
||||
// Partially resolved — demote to info with a note.
|
||||
findings.push({
|
||||
check: "inference",
|
||||
severity: "info",
|
||||
summary: `cloud gap partially resolved by repo grep: ${summary.slice(0, 120)}`,
|
||||
evidence: [
|
||||
`location: ${location.slice(0, 140)}`,
|
||||
`resolved via grep: ${resolved.join(",")}`,
|
||||
`unresolved: ${symbols.filter(s => !resolved.includes(s)).join(",")}`,
|
||||
],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
findings.push({
|
||||
check: "inference",
|
||||
severity: "warn",
|
||||
summary: `cloud-flagged gap not in any claim: ${summary.slice(0, 120)}`,
|
||||
evidence: [`location: ${location.slice(0, 140)}`],
|
||||
});
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
// Single cloud call — the consensus loop calls this N times in
|
||||
// parallel. Returns the parsed JSON shape + token usage + any error
|
||||
// diagnostic. NEVER throws; the consensus aggregator handles partial
|
||||
// failures by dropping non-parsed runs from the vote.
|
||||
interface CloudRunResult {
|
||||
parsed: any | null;
|
||||
tokens: number;
|
||||
error?: string; // "unreachable" | "non_200" | "unparseable"
|
||||
diagnostic?: string; // first 200 chars for debugging
|
||||
model: string;
|
||||
}
|
||||
|
||||
async function runCloudInference(systemMsg: string, userMsg: string, model: string): Promise<CloudRunResult> {
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(`${GATEWAY}/v1/chat`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
provider: "ollama_cloud",
|
||||
model,
|
||||
messages: [
|
||||
{ role: "system", content: systemMsg },
|
||||
{ role: "user", content: userMsg },
|
||||
],
|
||||
// temp=0 (greedy) + think=true. think=true is required for
|
||||
// gpt-oss:120b — without it the model returns empty content
|
||||
// on large prompts. Variance from the think trace is observed
|
||||
// in practice, which is why we use N=3 consensus, not single-
|
||||
// call determinism.
|
||||
max_tokens: 3000,
|
||||
temperature: 0,
|
||||
think: true,
|
||||
}),
|
||||
signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
|
||||
});
|
||||
} catch (e) {
|
||||
return { parsed: null, tokens: 0, error: "unreachable", diagnostic: (e as Error).message.slice(0, 200), model };
|
||||
}
|
||||
if (!resp.ok) {
|
||||
return { parsed: null, tokens: 0, error: "non_200", diagnostic: `${resp.status}: ${(await resp.text()).slice(0, 160)}`, model };
|
||||
}
|
||||
let body: any;
|
||||
try { body = await resp.json(); }
|
||||
catch (e) { return { parsed: null, tokens: 0, error: "unparseable", diagnostic: (e as Error).message, model }; }
|
||||
const content: string = body?.choices?.[0]?.message?.content ?? "";
|
||||
const tokens: number = body?.usage?.total_tokens ?? 0;
|
||||
const parsed = extractJson(content);
|
||||
if (!parsed) {
|
||||
return { parsed: null, tokens, error: "unparseable", diagnostic: content.slice(0, 200), model };
|
||||
}
|
||||
return { parsed, tokens, model };
|
||||
}
|
||||
|
||||
async function persistDiscrepancies(ctx: InferenceContext, discrepancies: any[]): Promise<void> {
|
||||
await mkdir("/home/profit/lakehouse/data/_kb", { recursive: true });
|
||||
const rows = discrepancies.map(d => JSON.stringify({
|
||||
pr_number: ctx.pr_number,
|
||||
head_sha: ctx.head_sha,
|
||||
logged_at: new Date().toISOString(),
|
||||
...d,
|
||||
}));
|
||||
await appendFile(AUDIT_DISCREPANCIES_JSONL, rows.join("\n") + "\n");
|
||||
}
|
||||
|
||||
// Extract structured knowledge from the curated scratchpad and append
|
||||
// to data/_kb/audit_facts.jsonl — one row per extract run, keyed by
|
||||
// PR number + head SHA for scope tracking. kb_query tails this next
|
||||
// audit to surface recurring entities/relationships across PRs.
|
||||
async function extractAndPersistFacts(scratchpad: string, ctx: InferenceContext): Promise<void> {
|
||||
const ex = await extractFacts(scratchpad);
|
||||
if (ex.error && ex.entities.length === 0 && ex.facts.length === 0) {
|
||||
// Full failure — log but don't write an empty row.
|
||||
console.error(`[inference] extractFacts skipped row: ${ex.error}`);
|
||||
return;
|
||||
}
|
||||
const row = {
|
||||
pr_number: ctx.pr_number,
|
||||
head_sha: ctx.head_sha,
|
||||
extracted_at: ex.extracted_at,
|
||||
extractor: ex.extractor_model,
|
||||
verifier: ex.verifier_model,
|
||||
llm_team_run_id: ex.llm_team_run_id ?? null,
|
||||
facts: ex.facts,
|
||||
entities: ex.entities,
|
||||
relationships: ex.relationships,
|
||||
verification_preview: ex.verification.slice(0, 400),
|
||||
verifier_verdicts: ex.verifier_verdicts,
|
||||
facts_dropped_by_verifier: ex.facts_dropped_by_verifier ?? 0,
|
||||
schema_version: 2,
|
||||
source: "audit_inference",
|
||||
};
|
||||
await mkdir("/home/profit/lakehouse/data/_kb", { recursive: true });
|
||||
await appendFile(AUDIT_FACTS_JSONL, JSON.stringify(row) + "\n");
|
||||
}
|
||||
|
||||
// Curation via tree-split — ports the scrum_master pattern into the
|
||||
// inference check. Shards the raw diff into DIFF_SHARD_SIZE chunks,
|
||||
// summarizes each shard *against the claim-verification task* so the
|
||||
// summary preserves exactly what the cloud needs to judge claims
|
||||
// (function signatures, struct fields, deletions, new files), drops
|
||||
// everything else. Merges into a compact scratchpad.
|
||||
//
|
||||
// Cost: N cloud calls for the shard summaries + 1 cloud call for the
|
||||
// final verification = N+1 calls instead of 1. Mitigation: shards run
|
||||
// serially (not parallel) to keep gateway load bounded; summary calls
|
||||
// use max_tokens=400 so they're fast (~2s each on gpt-oss:120b).
|
||||
//
|
||||
// Determinism: each shard summary call uses temp=0 + think=true (same
|
||||
// as the top-level inference call), so identical input yields
|
||||
// identical scratchpad. The final verification call then sees a
|
||||
// stable scratchpad, giving stable verdicts.
|
||||
async function treeSplitDiff(
|
||||
fullDiff: string,
|
||||
claims: Claim[],
|
||||
): Promise<{ scratchpad: string; shards: number }> {
|
||||
const shards: Array<{ from: number; to: number; text: string }> = [];
|
||||
for (let i = 0; i < fullDiff.length; i += DIFF_SHARD_SIZE) {
|
||||
const end = Math.min(i + DIFF_SHARD_SIZE, fullDiff.length);
|
||||
shards.push({ from: i, to: end, text: fullDiff.slice(i, end) });
|
||||
}
|
||||
// Curate the claim list into a short form the summary prompt can
|
||||
// use to bias extraction toward relevant facts.
|
||||
const claimDigest = claims.map((c, i) =>
|
||||
`${i}. [${c.strength}] "${c.text.slice(0, 100)}"`
|
||||
).join("\n");
|
||||
|
||||
let scratchpad = "";
|
||||
for (const [si, shard] of shards.entries()) {
|
||||
const prompt = [
|
||||
`You are summarizing shard ${si + 1}/${shards.length} (chars ${shard.from}..${shard.to}) of a PR diff.`,
|
||||
`The downstream task will verify these ship-claims against the full-PR summary. Extract ONLY facts that could confirm or refute these claims:`,
|
||||
"",
|
||||
claimDigest,
|
||||
"",
|
||||
"Extract: new function/method signatures, struct fields, deletions, new files, wiring (function X calls Y), absence-of-implementation markers, TODO comments on added lines.",
|
||||
"Skip: comment-only edits, whitespace, import reordering, unrelated cosmetic changes.",
|
||||
"",
|
||||
"─────── shard diff ───────",
|
||||
shard.text,
|
||||
"─────── end shard ───────",
|
||||
"",
|
||||
"Output: up to 180 words of facts in bullet form. No prose preamble, no claim verdicts (that's for the downstream step).",
|
||||
].join("\n");
|
||||
|
||||
const r = await callCloud(prompt, 400);
|
||||
if (r.content) {
|
||||
scratchpad += `\n--- shard ${si + 1} (chars ${shard.from}..${shard.to}) ---\n${r.content.trim()}\n`;
|
||||
}
|
||||
}
|
||||
return { scratchpad: scratchpad.trim(), shards: shards.length };
|
||||
}
|
||||
|
||||
// Minimal cloud caller used only by treeSplitDiff — same gateway +
|
||||
// model as the top-level call, but think=false. Shards are small
|
||||
// (≤DIFF_SHARD_SIZE ~4500 chars) and the task is pure fact
|
||||
// extraction, not reasoning. think=true on the shards introduced
|
||||
// variance in reasoning traces that compounded across 23 calls into
|
||||
// a non-deterministic scratchpad (observed during curation
|
||||
// validation: same-SHA runs produced 5/7/8 final findings).
|
||||
// think=false on small prompts is stable — only breaks at the main
|
||||
// call's 10K+ prompt size, which keeps think=true.
|
||||
async function callCloud(prompt: string, maxTokens: number): Promise<{ content: string }> {
|
||||
try {
|
||||
const r = await fetch(`${GATEWAY}/v1/chat`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
provider: "ollama_cloud",
|
||||
model: MODEL,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
max_tokens: maxTokens,
|
||||
temperature: 0,
|
||||
think: false,
|
||||
}),
|
||||
signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
|
||||
});
|
||||
if (!r.ok) return { content: "" };
|
||||
const j: any = await r.json();
|
||||
return { content: j?.choices?.[0]?.message?.content ?? "" };
|
||||
} catch {
|
||||
return { content: "" };
|
||||
}
|
||||
}
|
||||
|
||||
// Pull out plausible code-symbol names from a summary string.
|
||||
// Matches:
|
||||
// - identifier with backticks: `foo_bar`
|
||||
// - identifier followed by parens: foo_bar()
|
||||
// - CamelCase types
|
||||
// - snake_case_functions
|
||||
// Filters out common English words that could be matched accidentally.
|
||||
const STOPWORDS = new Set([
|
||||
"not","the","and","for","this","that","with","but","are","was","has",
|
||||
"have","been","any","missing","implementation","diff","defined","never",
|
||||
"referenced","integrated","flow","code","file","some","only","when",
|
||||
]);
|
||||
function extractSymbols(text: string): string[] {
|
||||
const out = new Set<string>();
|
||||
// `backticked` symbols
|
||||
for (const m of text.matchAll(/`([A-Za-z_][A-Za-z0-9_]{2,})`/g)) out.add(m[1]);
|
||||
// foo() or foo_bar() calls
|
||||
for (const m of text.matchAll(/\b([A-Za-z_][A-Za-z0-9_]{2,})\s*\(/g)) out.add(m[1]);
|
||||
// CamelCase types (3+ chars, must start with uppercase)
|
||||
for (const m of text.matchAll(/\b([A-Z][A-Za-z0-9]{2,})\b/g)) out.add(m[1]);
|
||||
return Array.from(out).filter(s => !STOPWORDS.has(s.toLowerCase()));
|
||||
}
|
||||
|
||||
// Scan the repo for at least one definition of each symbol. Uses Bun's
|
||||
// Glob to walk TS/Rust/Python/JS sources; ignores node_modules, data/,
|
||||
// and target/. Skips files > 500KB — those are fixtures/snapshots that
|
||||
// won't contain a definition line and slurping them slows the audit.
|
||||
async function symbolsExistInRepo(symbols: string[]): Promise<string[]> {
|
||||
const patterns = ["**/*.ts", "**/*.tsx", "**/*.rs", "**/*.py", "**/*.js"];
|
||||
const skip = (p: string) => p.includes("/node_modules/") || p.startsWith("data/") || p.includes("/target/") || p.startsWith("dist/");
|
||||
const MAX_FILE_BYTES = 500_000;
|
||||
const { stat } = await import("node:fs/promises");
|
||||
const resolved = new Set<string>();
|
||||
const toFind = new Set(symbols);
|
||||
for (const pat of patterns) {
|
||||
if (toFind.size === 0) break;
|
||||
const glob = new Glob(pat);
|
||||
for await (const f of glob.scan({ cwd: REPO_ROOT, onlyFiles: true })) {
|
||||
if (skip(f)) continue;
|
||||
try { const s = await stat(`${REPO_ROOT}/${f}`); if (s.size > MAX_FILE_BYTES) continue; } catch { continue; }
|
||||
let content: string;
|
||||
try { content = await readFile(`${REPO_ROOT}/${f}`, "utf8"); } catch { continue; }
|
||||
for (const sym of Array.from(toFind)) {
|
||||
// Definition heuristics: `function sym`, `fn sym`, `const sym`,
|
||||
// `let sym`, `def sym`, `class sym`, `struct sym`, `enum sym`,
|
||||
// `trait sym`, `async function sym`, `pub (async )?fn sym`.
|
||||
const re = new RegExp(
|
||||
`\\b(function|async\\s+function|const|let|var|def|class|struct|enum|trait|impl|type|interface|fn|pub\\s+(async\\s+)?fn)\\s+${escapeRe(sym)}\\b`
|
||||
);
|
||||
if (re.test(content)) {
|
||||
resolved.add(sym);
|
||||
toFind.delete(sym);
|
||||
if (toFind.size === 0) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(resolved);
|
||||
}
|
||||
|
||||
function escapeRe(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
// Lift the first balanced JSON object out of the response. Tolerates
|
||||
// leading prose, code fences, and model reasoning preamble when the
|
||||
// cloud model ignored "strict JSON only."
|
||||
function extractJson(text: string): any | null {
|
||||
const cleaned = text.replace(/^```(?:json)?\s*/im, "").replace(/```\s*$/im, "");
|
||||
let depth = 0;
|
||||
let start = -1;
|
||||
for (let i = 0; i < cleaned.length; i++) {
|
||||
const c = cleaned[i];
|
||||
if (c === "{") {
|
||||
if (depth === 0) start = i;
|
||||
depth++;
|
||||
} else if (c === "}") {
|
||||
depth--;
|
||||
if (depth === 0 && start >= 0) {
|
||||
try { return JSON.parse(cleaned.slice(start, i + 1)); } catch { start = -1; }
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,397 @@
|
||||
// Local-KB check — reads data/_kb/ + data/_observer/ + data/_bot/
|
||||
// for prior evidence bearing on this PR's claims. Cheap, offline,
|
||||
// no model calls. The point: if a claim like "Phase X shipped" has
|
||||
// a historical record of failing on the same signature before, the
|
||||
// auditor surfaces that pattern before the cloud check has to
|
||||
// infer it.
|
||||
//
|
||||
// What this check reads (all file-backed, append-only or periodic):
|
||||
// data/_kb/outcomes.jsonl — per-scenario outcomes (kb.ts)
|
||||
// data/_kb/error_corrections.jsonl — fail→succeed deltas on same sig
|
||||
// data/_kb/scrum_reviews.jsonl — scrum-master accepted reviews
|
||||
// data/_observer/ops.jsonl — observer ring → disk stream
|
||||
// data/_bot/cycles/*.json — bot cycle results
|
||||
//
|
||||
// Each JSONL line / per-cycle file is small; this check reads tails
|
||||
// only (last N lines or last M files) to stay cheap on large corpora.
|
||||
|
||||
import { readFile, readdir, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Claim, Finding } from "../types.ts";
|
||||
import { aggregate, ratingSeverity, formatAgg } from "../kb_index.ts";
|
||||
|
||||
const KB_DIR = "/home/profit/lakehouse/data/_kb";
|
||||
const OBSERVER_OPS = "/home/profit/lakehouse/data/_observer/ops.jsonl";
|
||||
const BOT_CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
|
||||
const SCRUM_REVIEWS_JSONL = "/home/profit/lakehouse/data/_kb/scrum_reviews.jsonl";
|
||||
const AUDIT_LESSONS_JSONL = "/home/profit/lakehouse/data/_kb/audit_lessons.jsonl";
|
||||
const AUDIT_FACTS_JSONL = "/home/profit/lakehouse/data/_kb/audit_facts.jsonl";
|
||||
const TAIL_LINES = 500;
|
||||
const MAX_BOT_CYCLE_FILES = 30;
|
||||
|
||||
export async function runKbCheck(claims: Claim[], prFiles: string[] = []): Promise<Finding[]> {
|
||||
const findings: Finding[] = [];
|
||||
|
||||
// 1. Recent scenario outcomes: are strong-claim-style phrases showing
|
||||
// up alongside failed events? That's "we claimed it worked" +
|
||||
// "it didn't" in the KB.
|
||||
const scenarioFindings = await checkScenarioOutcomes(claims);
|
||||
findings.push(...scenarioFindings);
|
||||
|
||||
// 2. Error corrections: any of the claims text overlap a
|
||||
// recently-observed fail→succeed pair? If yes, add context.
|
||||
const correctionFindings = await checkErrorCorrections(claims);
|
||||
findings.push(...correctionFindings);
|
||||
|
||||
// 3. Bot cycles: any prior bot cycle ended in tests_failed or
|
||||
// apply_failed on a file this PR is also touching?
|
||||
const botFindings = await checkBotCycles();
|
||||
findings.push(...botFindings);
|
||||
|
||||
// 4. Observer: count recent error events. High volume = shared
|
||||
// infra problem, worth flagging (context for other findings).
|
||||
const obsFindings = await checkObserverStream();
|
||||
findings.push(...obsFindings);
|
||||
|
||||
// 5. Scrum-master reviews — surface prior accepted reviews for any
|
||||
// file in this PR's diff. Cohesion plan Phase C wire: the
|
||||
// auditor gets to "borrow" the scrum-master's deeper per-file
|
||||
// analysis instead of re-doing that work.
|
||||
if (prFiles.length > 0) {
|
||||
const scrumFindings = await checkScrumReviews(prFiles);
|
||||
findings.push(...scrumFindings);
|
||||
}
|
||||
|
||||
// 6b. Audit-facts (llm_team extract pipeline output) — surface
|
||||
// entities that recur across multiple PRs. These are the
|
||||
// "core system entities" accumulating in the knowledge base;
|
||||
// showing them as info on future audits gives reviewers
|
||||
// architectural context the raw diff doesn't convey.
|
||||
const factFindings = await checkAuditFacts();
|
||||
findings.push(...factFindings);
|
||||
|
||||
// 6. Audit-lessons feedback loop — summarize the top recurring
|
||||
// patterns from prior audits' block/warn findings. If the same
|
||||
// pattern signature has fired 3+ times across prior audits,
|
||||
// emit it as a block-severity finding so reviewers know this
|
||||
// is a known-recurring class, not a one-off. Does NOT couple
|
||||
// to the current audit's static/inference findings (those run
|
||||
// in parallel and we can't see them here) — the amplification
|
||||
// is emergent: if the current audit's finding-summary matches
|
||||
// a top recurrence, the reviewer sees both.
|
||||
const auditLessonFindings = await checkAuditLessons();
|
||||
findings.push(...auditLessonFindings);
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function tailJsonl<T = any>(path: string, n: number): Promise<T[]> {
|
||||
try {
|
||||
const raw = await readFile(path, "utf8");
|
||||
const lines = raw.split("\n").filter(l => l.length > 0);
|
||||
const slice = lines.slice(-n);
|
||||
const out: T[] = [];
|
||||
for (const line of slice) {
|
||||
try { out.push(JSON.parse(line)); } catch { /* skip malformed */ }
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function checkScenarioOutcomes(_claims: Claim[]): Promise<Finding[]> {
|
||||
const outcomes = await tailJsonl<any>(join(KB_DIR, "outcomes.jsonl"), TAIL_LINES);
|
||||
if (outcomes.length === 0) return [];
|
||||
const totalEvents = outcomes.reduce((s, o) => s + (o.total_events ?? 0), 0);
|
||||
const okEvents = outcomes.reduce((s, o) => s + (o.ok_events ?? 0), 0);
|
||||
const failRate = totalEvents > 0 ? 1 - okEvents / totalEvents : 0;
|
||||
|
||||
if (totalEvents === 0) {
|
||||
return [{
|
||||
check: "kb_query",
|
||||
severity: "info",
|
||||
summary: `KB: no scenario outcomes on file — learning loop is empty`,
|
||||
evidence: [`data/_kb/outcomes.jsonl has ${outcomes.length} entries with 0 total events`],
|
||||
}];
|
||||
}
|
||||
|
||||
const recent = outcomes.slice(-10);
|
||||
const recentFailSigs: string[] = recent
|
||||
.filter(o => (o.ok_events ?? 0) < (o.total_events ?? 0))
|
||||
.map(o => o.sig_hash)
|
||||
.filter(s => typeof s === "string");
|
||||
|
||||
const findings: Finding[] = [{
|
||||
check: "kb_query",
|
||||
severity: failRate > 0.3 ? "warn" : "info",
|
||||
summary: `KB: ${outcomes.length} recent scenario runs, ${okEvents}/${totalEvents} events ok (fail rate ${(failRate * 100).toFixed(1)}%)`,
|
||||
evidence: [
|
||||
`most recent: ${recent[recent.length - 1]?.run_id ?? "?"}`,
|
||||
`recent failing sigs: ${recentFailSigs.length > 0 ? recentFailSigs.slice(-3).join(", ") : "none"}`,
|
||||
],
|
||||
}];
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function checkErrorCorrections(_claims: Claim[]): Promise<Finding[]> {
|
||||
const corrections = await tailJsonl<any>(join(KB_DIR, "error_corrections.jsonl"), TAIL_LINES);
|
||||
if (corrections.length === 0) return [];
|
||||
return [{
|
||||
check: "kb_query",
|
||||
severity: "info",
|
||||
summary: `KB: ${corrections.length} error corrections on file (fail→succeed pairs)`,
|
||||
evidence: [
|
||||
corrections.length > 0
|
||||
? `most recent: ${String(corrections[corrections.length - 1]?.sig_hash ?? "?").slice(0, 24)}`
|
||||
: "none",
|
||||
],
|
||||
}];
|
||||
}
|
||||
|
||||
async function checkBotCycles(): Promise<Finding[]> {
|
||||
let entries: string[] = [];
|
||||
try { entries = await readdir(BOT_CYCLES_DIR); }
|
||||
catch { return []; }
|
||||
|
||||
const jsonFiles = entries.filter(e => e.endsWith(".json"));
|
||||
if (jsonFiles.length === 0) return [];
|
||||
|
||||
// Sort by mtime desc, take most recent N
|
||||
const withStat = await Promise.all(
|
||||
jsonFiles.map(async name => {
|
||||
try { return { name, mtime: (await stat(join(BOT_CYCLES_DIR, name))).mtimeMs }; }
|
||||
catch { return { name, mtime: 0 }; }
|
||||
}),
|
||||
);
|
||||
const recent = withStat.sort((a, b) => b.mtime - a.mtime).slice(0, MAX_BOT_CYCLE_FILES);
|
||||
|
||||
const outcomes: Record<string, number> = {};
|
||||
for (const { name } of recent) {
|
||||
try {
|
||||
const r = JSON.parse(await readFile(join(BOT_CYCLES_DIR, name), "utf8"));
|
||||
const o = String(r.outcome ?? "unknown");
|
||||
outcomes[o] = (outcomes[o] ?? 0) + 1;
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
|
||||
const summary = Object.entries(outcomes)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(", ");
|
||||
|
||||
const failCount = (outcomes["tests_failed"] ?? 0) + (outcomes["apply_failed"] ?? 0) + (outcomes["model_failed"] ?? 0);
|
||||
return [{
|
||||
check: "kb_query",
|
||||
severity: failCount > recent.length / 2 ? "warn" : "info",
|
||||
summary: `KB: bot recorded ${recent.length} recent cycles — ${summary || "no outcomes parsed"}`,
|
||||
evidence: [
|
||||
`dir: ${BOT_CYCLES_DIR}`,
|
||||
`fail-class total: ${failCount} / ${recent.length}`,
|
||||
],
|
||||
}];
|
||||
}
|
||||
|
||||
async function checkObserverStream(): Promise<Finding[]> {
|
||||
const ops = await tailJsonl<any>(OBSERVER_OPS, TAIL_LINES);
|
||||
if (ops.length === 0) return [];
|
||||
const failures = ops.filter(o => o.ok === false).length;
|
||||
return [{
|
||||
check: "kb_query",
|
||||
severity: "info",
|
||||
summary: `KB: observer stream ${ops.length} recent ops, ${failures} failures`,
|
||||
evidence: [
|
||||
`source: ${OBSERVER_OPS}`,
|
||||
`by source: ${observerBySource(ops)}`,
|
||||
],
|
||||
}];
|
||||
}
|
||||
|
||||
function observerBySource(ops: any[]): string {
|
||||
const c: Record<string, number> = {};
|
||||
for (const o of ops) {
|
||||
const s = String(o.source ?? "unknown");
|
||||
c[s] = (c[s] ?? 0) + 1;
|
||||
}
|
||||
return Object.entries(c).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(", ") || "empty";
|
||||
}
|
||||
|
||||
// Audit-facts — reads data/_kb/audit_facts.jsonl (populated by every
|
||||
// curated inference run via llm_team's extract pipeline). Each row
|
||||
// has arrays: facts, entities, relationships. We explode entities and
|
||||
// aggregate them across PRs using kb_index. An entity seen in 3+ PRs
|
||||
// is a "core system entity" — we surface the top N as info context.
|
||||
//
|
||||
// Filters out short names (<3 chars, likely qwen2.5 truncation
|
||||
// artifacts) and generic types ("string", "number") that would
|
||||
// otherwise dominate the ranking.
|
||||
const ENTITY_NAME_MIN_LEN = 3;
|
||||
const GENERIC_ENTITY_NAMES = new Set([
|
||||
"string", "number", "boolean", "any", "void", "unknown", "never",
|
||||
"object", "array", "function", "const", "let", "var", "true", "false",
|
||||
"null", "undefined", "promise", "map", "set", "record",
|
||||
]);
|
||||
|
||||
async function checkAuditFacts(): Promise<Finding[]> {
|
||||
// Read raw rows — each row has multiple entities, so we can't just
|
||||
// use aggregate() directly (it's one-signature-per-row). Explode
|
||||
// entities into (row, entity) pairs, then aggregate by entity name.
|
||||
let raw: string;
|
||||
try { raw = await (await import("node:fs/promises")).readFile(AUDIT_FACTS_JSONL, "utf8"); }
|
||||
catch { return []; }
|
||||
const lines = raw.split("\n").filter(l => l.length > 0);
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
interface EntityRow { entity_key: string; pr_number: number; type: string; name: string; description: string }
|
||||
const entityRows: EntityRow[] = [];
|
||||
for (const line of lines.slice(-TAIL_LINES * 2)) {
|
||||
let row: any;
|
||||
try { row = JSON.parse(line); } catch { continue; }
|
||||
const prNum = Number(row?.pr_number);
|
||||
if (!Number.isFinite(prNum)) continue;
|
||||
for (const e of Array.isArray(row?.entities) ? row.entities : []) {
|
||||
const name = String(e?.name ?? "").trim();
|
||||
if (name.length < ENTITY_NAME_MIN_LEN) continue;
|
||||
if (GENERIC_ENTITY_NAMES.has(name.toLowerCase())) continue;
|
||||
entityRows.push({
|
||||
entity_key: name.toLowerCase(),
|
||||
pr_number: prNum,
|
||||
type: String(e?.type ?? "?"),
|
||||
name,
|
||||
description: String(e?.description ?? "").slice(0, 160),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (entityRows.length === 0) return [];
|
||||
|
||||
// Aggregate manually — one key per entity name, distinct_scopes by PR.
|
||||
type Agg = { count: number; scopes: Set<number>; types: Set<string>; last_name: string; last_desc: string };
|
||||
const byEntity = new Map<string, Agg>();
|
||||
for (const r of entityRows) {
|
||||
const a = byEntity.get(r.entity_key) ?? {
|
||||
count: 0, scopes: new Set<number>(), types: new Set<string>(), last_name: "", last_desc: "",
|
||||
};
|
||||
a.count += 1;
|
||||
a.scopes.add(r.pr_number);
|
||||
a.types.add(r.type);
|
||||
a.last_name = r.name;
|
||||
a.last_desc = r.description;
|
||||
byEntity.set(r.entity_key, a);
|
||||
}
|
||||
|
||||
// Rank: require 2+ distinct PRs (same-PR entity-repeats don't count
|
||||
// as "cross-cutting"). Take the top 5 to avoid flooding the verdict.
|
||||
const ranked = Array.from(byEntity.entries())
|
||||
.filter(([_, a]) => a.scopes.size >= 2)
|
||||
.sort((a, b) => b[1].scopes.size - a[1].scopes.size || b[1].count - a[1].count)
|
||||
.slice(0, 5);
|
||||
|
||||
if (ranked.length === 0) {
|
||||
// Useful to know the KB is being populated — emit a single
|
||||
// summary so operators see fact extraction is alive.
|
||||
return [{
|
||||
check: "kb_query",
|
||||
severity: "info",
|
||||
summary: `audit_facts KB has ${entityRows.length} entity-observations across ${new Set(entityRows.map(r => r.pr_number)).size} PRs (no cross-PR recurrences yet)`,
|
||||
evidence: [`source: ${AUDIT_FACTS_JSONL}`],
|
||||
}];
|
||||
}
|
||||
|
||||
return ranked.map(([_, a]) => ({
|
||||
check: "kb_query" as const,
|
||||
severity: "info" as const,
|
||||
summary: `core entity \`${a.last_name}\` recurs in ${a.scopes.size} PRs (types: ${Array.from(a.types).join(",")})`,
|
||||
evidence: [
|
||||
`count=${a.count} distinct_PRs=${a.scopes.size}`,
|
||||
`description: ${a.last_desc.slice(0, 200)}`,
|
||||
`PRs: ${Array.from(a.scopes).sort((x, y) => x - y).join(",")}`,
|
||||
],
|
||||
}));
|
||||
}
|
||||
|
||||
// Audit-lessons — reads data/_kb/audit_lessons.jsonl (populated by
|
||||
// every audit's appendAuditLessons). Uses the shared kb_index
|
||||
// aggregator: groups by `signature`, distinct-scopes keyed by PR
|
||||
// number, severity from ratingSeverity(agg) which applies the
|
||||
// confidence × count rating (see kb_index.ts). This is the same
|
||||
// aggregation any other KB reader uses — shared discipline, not
|
||||
// per-check custom logic.
|
||||
async function checkAuditLessons(): Promise<Finding[]> {
|
||||
const bySig = await aggregate<any>(AUDIT_LESSONS_JSONL, {
|
||||
keyFn: (r) => r?.signature,
|
||||
scopeFn: (r) => (r?.pr_number !== undefined ? `pr-${r.pr_number}` : undefined),
|
||||
checkFn: (r) => r?.check,
|
||||
tailLimit: TAIL_LINES * 4,
|
||||
});
|
||||
if (bySig.size === 0) return [];
|
||||
|
||||
const findings: Finding[] = [];
|
||||
for (const [sig, agg] of bySig) {
|
||||
// Silent on first-ever occurrence — not yet signal.
|
||||
if (agg.count < 2) continue;
|
||||
const sev = ratingSeverity(agg);
|
||||
findings.push({
|
||||
check: "kb_query",
|
||||
severity: sev,
|
||||
summary: `recurring audit pattern (${agg.distinct_scopes} distinct PRs, ${agg.count} flaggings, conf=${agg.confidence.toFixed(2)}): ${agg.representative_summary.slice(0, 160)}`,
|
||||
evidence: [
|
||||
`signature=${sig}`,
|
||||
`checks: ${agg.checks.join(",")}`,
|
||||
`scopes: ${agg.scopes.slice(-6).join(",")}`,
|
||||
formatAgg(agg),
|
||||
],
|
||||
});
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
// Scrum-master reviews — the scrum pipeline writes one row per
|
||||
// accepted per-file review. We match reviews whose `file` matches
|
||||
// any path in the PR's diff, then surface the *preview* + which
|
||||
// model the escalation ladder had to reach. If the scrum-master
|
||||
// needed the 123B specialist or larger to resolve a file, that's
|
||||
// a meaningful signal about the code's complexity — and it's
|
||||
// surfaced to the PR without the auditor having to re-run the
|
||||
// escalation ladder itself.
|
||||
async function checkScrumReviews(prFiles: string[]): Promise<Finding[]> {
|
||||
const rows = await tailJsonl<any>(SCRUM_REVIEWS_JSONL, TAIL_LINES);
|
||||
if (rows.length === 0) return [];
|
||||
|
||||
// Match by exact file OR filename suffix — PR files arrive as
|
||||
// `auditor/audit.ts`-style relative paths; scrum stores the same.
|
||||
const norm = (p: string) => p.replace(/^\/+/, "").replace(/^home\/profit\/lakehouse\//, "");
|
||||
const prSet = new Set(prFiles.map(norm));
|
||||
|
||||
// Keep only the most recent review per file (last-wins).
|
||||
const latestByFile = new Map<string, any>();
|
||||
for (const r of rows) {
|
||||
const f = norm(String(r.file ?? ""));
|
||||
if (!f) continue;
|
||||
if (!prSet.has(f)) continue;
|
||||
latestByFile.set(f, r);
|
||||
}
|
||||
if (latestByFile.size === 0) return [];
|
||||
|
||||
const findings: Finding[] = [];
|
||||
for (const [file, r] of latestByFile) {
|
||||
const model = String(r.accepted_model ?? "?");
|
||||
const attempt = r.accepted_on_attempt ?? "?";
|
||||
const treeSplit = !!r.tree_split_fired;
|
||||
// Heuristic: if the scrum-master had to escalate past attempt 3,
|
||||
// or had to tree-split, that's context the PR reviewer should see.
|
||||
// Severity: info for low-escalation, warn if escalated far up
|
||||
// the ladder (cloud specialist required).
|
||||
const heavyEscalation = Number(attempt) >= 4;
|
||||
const sev: "warn" | "info" = heavyEscalation ? "warn" : "info";
|
||||
findings.push({
|
||||
check: "kb_query",
|
||||
severity: sev,
|
||||
summary: `scrum-master review for \`${file}\` — accepted on attempt ${attempt} by \`${model}\`${treeSplit ? " (tree-split)" : ""}`,
|
||||
evidence: [
|
||||
`reviewed_at: ${r.reviewed_at ?? "?"}`,
|
||||
`preview: ${String(r.suggestions_preview ?? "").slice(0, 300).replace(/\n/g, " ")}`,
|
||||
],
|
||||
});
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// Static diff check — grep-style, no AST, no LLM. Looks for patterns
|
||||
// that are high-signal evidence of placeholder code.
|
||||
//
|
||||
// Findings are severity-graded:
|
||||
// block — explicit non-impl markers (unimplemented!, todo!,
|
||||
// panic!("not implemented"), throw new Error("not implemented"))
|
||||
// warn — TODO / FIXME / XXX / HACK comments on added lines,
|
||||
// new struct fields with no read-site anywhere in the diff,
|
||||
// suspiciously-empty function bodies ({ Ok(()) } / {} when
|
||||
// the commit message claims the fn "implements" something)
|
||||
// info — hardcoded "test" / "dummy" / "placeholder" strings in
|
||||
// added lines (could be real, just flag for inspection)
|
||||
//
|
||||
// Consumes: raw unified diff text from Gitea.
|
||||
|
||||
import type { Finding } from "../types.ts";
|
||||
|
||||
// Rust + TypeScript patterns that almost always indicate "this is
|
||||
// not actually implemented yet."
|
||||
const BLOCK_PATTERNS: Array<{ re: RegExp; why: string }> = [
|
||||
{ re: /\bunimplemented!\s*\(/, why: "unimplemented!() macro call" },
|
||||
{ re: /\btodo!\s*\(/, why: "todo!() macro call" },
|
||||
{ re: /panic!\s*\(\s*"(?:not implemented|TODO|not yet|unimpl)/i, why: "panic! with not-implemented message" },
|
||||
{ re: /throw\s+new\s+Error\s*\(\s*['"](?:not implemented|TODO|unimpl)/i, why: "throw Error 'not implemented'" },
|
||||
];
|
||||
|
||||
const WARN_COMMENT_PATTERNS: Array<{ re: RegExp; why: string }> = [
|
||||
{ re: /^\+.*\/\/\s*(TODO|FIXME|XXX|HACK)\b/i, why: "TODO/FIXME/XXX/HACK comment added" },
|
||||
{ re: /^\+.*#\s*(TODO|FIXME|XXX|HACK)\b/i, why: "TODO/FIXME/XXX/HACK comment added" },
|
||||
];
|
||||
|
||||
const INFO_HARDCODED_PATTERNS: Array<{ re: RegExp; why: string }> = [
|
||||
{ re: /"(?:placeholder|dummy|foobar|xxx|replaceme|changeme)"/i, why: "suspicious hardcoded string" },
|
||||
];
|
||||
|
||||
export function runStaticCheck(diff: string): Finding[] {
|
||||
const findings: Finding[] = [];
|
||||
|
||||
// Per-file walk: only look at ADDED lines (prefix '+' but not '+++'
|
||||
// which is the diff header).
|
||||
const perFile = splitDiffByFile(diff);
|
||||
|
||||
for (const [path, lines] of perFile) {
|
||||
// Skip diff bookkeeping + pure-delete files
|
||||
if (!lines.some(l => l.startsWith("+") && !l.startsWith("+++"))) continue;
|
||||
|
||||
// The auditor's own check files literally contain the BLOCK
|
||||
// patterns as regex definitions (BLOCK_PATTERNS in this file,
|
||||
// prompt examples in inference.ts). Skipping BLOCK scan on these
|
||||
// specific paths prevents the checker from self-flagging its own
|
||||
// string literals. WARN/INFO patterns still run — those genuinely
|
||||
// could indicate problems in the checker's own code (TODO
|
||||
// comments don't self-define).
|
||||
const isAuditorCheckerFile = path.startsWith("auditor/checks/") ||
|
||||
path.startsWith("auditor/fixtures/");
|
||||
|
||||
for (let idx = 0; idx < lines.length; idx++) {
|
||||
const line = lines[idx];
|
||||
if (!line.startsWith("+") || line.startsWith("+++")) continue;
|
||||
const added = line.slice(1);
|
||||
|
||||
if (!isAuditorCheckerFile) {
|
||||
for (const { re, why } of BLOCK_PATTERNS) {
|
||||
const m = added.match(re);
|
||||
if (m && typeof m.index === "number") {
|
||||
// Skip if the match sits inside a quoted string literal —
|
||||
// this is how rubric files (tests/real-world/*, prompt
|
||||
// templates) legitimately reference the patterns they
|
||||
// guard against, without actually executing them.
|
||||
if (isInsideQuotedString(added, m.index)) continue;
|
||||
findings.push({
|
||||
check: "static",
|
||||
severity: "block",
|
||||
summary: `${why} in ${path}`,
|
||||
evidence: [`${path}:+${idx + 1}: ${added.trim().slice(0, 160)}`],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const { re, why } of WARN_COMMENT_PATTERNS) {
|
||||
if (re.test(line)) {
|
||||
findings.push({
|
||||
check: "static",
|
||||
severity: "warn",
|
||||
summary: `${why} in ${path}`,
|
||||
evidence: [`${path}:+${idx + 1}: ${added.trim().slice(0, 160)}`],
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const { re, why } of INFO_HARDCODED_PATTERNS) {
|
||||
if (re.test(added)) {
|
||||
findings.push({
|
||||
check: "static",
|
||||
severity: "info",
|
||||
summary: `${why} in ${path}`,
|
||||
evidence: [`${path}:+${idx + 1}: ${added.trim().slice(0, 160)}`],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// "Field added but never read" heuristic — catches exactly the
|
||||
// Phase 45 DocRef placeholder pattern. Limited to the diff itself:
|
||||
// we're not doing a full-codebase grep here (too noisy; callers
|
||||
// elsewhere might exist). The point is: if NEITHER this diff nor
|
||||
// any other line in the diff reads the field, the PR is shipping
|
||||
// state without a consumer.
|
||||
const addedLines = lines.filter(l => l.startsWith("+") && !l.startsWith("+++"))
|
||||
.map(l => l.slice(1));
|
||||
const newFields = extractNewFields(addedLines);
|
||||
for (const field of newFields) {
|
||||
const readPattern = new RegExp(`[\\.:]\\s*${escape(field)}\\b|\\b${escape(field)}\\s*:`);
|
||||
// The definition line itself matches readPattern — filter it out
|
||||
// by requiring at least TWO lines in the diff mention the field
|
||||
// (one defines, one reads).
|
||||
const hits = addedLines.filter(l => readPattern.test(l));
|
||||
if (hits.length < 2) {
|
||||
findings.push({
|
||||
check: "static",
|
||||
severity: "warn",
|
||||
summary: `field '${field}' added in ${path} but no read-site in the diff — could be placeholder state without a consumer`,
|
||||
evidence: [`${path}: added '${field}' with no reader; rest of diff has ${hits.length - 1} mentions`],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
function splitDiffByFile(diff: string): Map<string, string[]> {
|
||||
const out = new Map<string, string[]>();
|
||||
let current: string | null = null;
|
||||
let buf: string[] = [];
|
||||
for (const line of diff.split(/\r?\n/)) {
|
||||
const m = line.match(/^diff --git a\/(\S+) b\/(\S+)/);
|
||||
if (m) {
|
||||
if (current) out.set(current, buf);
|
||||
current = m[2];
|
||||
buf = [];
|
||||
continue;
|
||||
}
|
||||
buf.push(line);
|
||||
}
|
||||
if (current) out.set(current, buf);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Extract new `pub name: Type,` fields from added lines. Rust syntax.
|
||||
// Narrowly-scoped: only matches at the start of a trimmed line,
|
||||
// requires `pub ` prefix, ignores `pub fn` / `pub struct` / etc.
|
||||
function extractNewFields(addedLines: string[]): string[] {
|
||||
const fields = new Set<string>();
|
||||
for (const line of addedLines) {
|
||||
const t = line.trim();
|
||||
// pub NAME: Type,
|
||||
const m = t.match(/^pub\s+(?!fn\b|struct\b|enum\b|mod\b|use\b|trait\b|impl\b|const\b|static\b|type\b)(\w+)\s*:/);
|
||||
if (m) fields.add(m[1]);
|
||||
}
|
||||
return Array.from(fields);
|
||||
}
|
||||
|
||||
// True if `pos` falls inside a double- or single-quoted string on this
|
||||
// line (backtick template literals too). Walks left→right toggling the
|
||||
// "in quote" state on each unescaped quote. Good enough for single-
|
||||
// line matches; multi-line strings aren't parsed (they're extremely
|
||||
// rare in the patterns we're blocking on, and would require a proper
|
||||
// tokenizer to handle correctly).
|
||||
function isInsideQuotedString(line: string, pos: number): boolean {
|
||||
let inDouble = false, inSingle = false, inBacktick = false;
|
||||
for (let i = 0; i < pos; i++) {
|
||||
const c = line[i];
|
||||
const esc = i > 0 && line[i - 1] === "\\";
|
||||
if (esc) continue;
|
||||
if (c === '"' && !inSingle && !inBacktick) inDouble = !inDouble;
|
||||
else if (c === "'" && !inDouble && !inBacktick) inSingle = !inSingle;
|
||||
else if (c === "`" && !inDouble && !inSingle) inBacktick = !inBacktick;
|
||||
}
|
||||
return inDouble || inSingle || inBacktick;
|
||||
}
|
||||
|
||||
function escape(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
// Claim parser — reads commit messages + PR body, extracts ship-claims.
|
||||
//
|
||||
// A "ship-claim" is any phrase that asserts functionality is working,
|
||||
// tested, complete, or landed. These are the assertions the downstream
|
||||
// checks (static/dynamic/inference/kb) try to falsify.
|
||||
//
|
||||
// Heuristic approach (regex + strength grading) — intentionally NOT
|
||||
// using an LLM here. Reason: the inference check already asks a cloud
|
||||
// model "does this match the claim?". The parser's job is to surface
|
||||
// the claim substrates, not judge them. Over-engineering the parser
|
||||
// risks false-negatives when the cloud model was going to catch it
|
||||
// anyway.
|
||||
|
||||
import type { Claim, PrSnapshot } from "./types.ts";
|
||||
|
||||
// Strong claims: explicit end-to-end + verification vocabulary
|
||||
const STRONG_PATTERNS: RegExp[] = [
|
||||
/\bverified\s+(end[- ]to[- ]end|live|in\s+production|against)\b/i,
|
||||
/\btested\s+(live|end[- ]to[- ]end|against|with)\b/i,
|
||||
/\bworks\s+(end[- ]to[- ]end|live|in\s+production)\b/i,
|
||||
/\bproduction[- ]ready\b/i,
|
||||
/\bfully\s+(functional|wired|working)\b/i,
|
||||
/\bphase\s+\d+(\.\d+)?\s+(shipped|complete|done|landed)\b/i,
|
||||
/\bground\s+truth\b/i,
|
||||
/\bproven\b/i,
|
||||
];
|
||||
|
||||
// Moderate claims: asserted completion or pass but without the strong
|
||||
// verification qualifier.
|
||||
const MODERATE_PATTERNS: RegExp[] = [
|
||||
/\bshipped\b/i,
|
||||
/\blanded\b/i,
|
||||
/\bgreen\b/i,
|
||||
/\b(tests?\s+)?pass(ing|ed)\b/i,
|
||||
/\bcomplet(e|ed)\b/i,
|
||||
/\bdone\b/i,
|
||||
/\bwired\b/i,
|
||||
/\bfixed\b/i,
|
||||
/\bworks\b/i,
|
||||
];
|
||||
|
||||
// Weak claims: aspirational or hedged. Usually low-risk but recorded
|
||||
// for completeness.
|
||||
const WEAK_PATTERNS: RegExp[] = [
|
||||
/\bshould\s+work\b/i,
|
||||
/\bexpected\s+to\b/i,
|
||||
/\bintended\s+to\b/i,
|
||||
/\bwill\s+(work|handle|support)\b/i,
|
||||
/\bprobably\b/i,
|
||||
];
|
||||
|
||||
// Empirical claims: runtime measurements / observed outcomes that can't
|
||||
// be verified from a diff (only from the actual run that produced
|
||||
// them). Classifying as empirical lets the inference check skip
|
||||
// diff-verification and saves the ladder for falsifiable claims.
|
||||
//
|
||||
// Two classes share this bucket because they share the skip discipline:
|
||||
//
|
||||
// 1. Runtime metrics — "58 cloud calls", "306s end-to-end"
|
||||
// 2. History/proof refs — "verified on PR #8", "was flipping across runs"
|
||||
//
|
||||
// Both are assertions about state outside the current diff. The cloud
|
||||
// would flag them as "not backed" — but that's a false positive: the
|
||||
// proof lives in the referenced run, prior commit, or test output, not
|
||||
// in the added lines the cloud is reading.
|
||||
const EMPIRICAL_PATTERNS: RegExp[] = [
|
||||
// ─── Runtime metrics ───
|
||||
// Iteration / attempt counts: "6/6 iterations", "attempt 5", "accepted on attempt 3"
|
||||
/\b\d+\s*\/\s*\d+\s+(iterations?|attempts?|cycles?|runs?|shards?)\b/i,
|
||||
/\b(accepted|resolved|converged)\s+on\s+attempt\s+\d+\b/i,
|
||||
// Runtime metrics: "58 cloud calls", "306s end-to-end", "245s total", "5931 chars"
|
||||
/\b\d+\s+(cloud\s+)?calls?\b/i,
|
||||
/\b\d+\s*(ms|s|seconds?|minutes?|m)\s+(end[- ]to[- ]end|total|elapsed|duration)\b/i,
|
||||
/\b\d+\s+chars?\b.*\b(accepted|generated|produced)\b/i,
|
||||
// "escalated through N tiers", "N distinct models"
|
||||
/\bescalated\s+through\s+\d+\b/i,
|
||||
/\b\d+\s+distinct\s+(model|tier)s?\b/i,
|
||||
|
||||
// ─── History / proof references ───
|
||||
// "verified on PR #8", "verified end-to-end on PR 8", "tested against PR #4"
|
||||
// Require PR#N / commit-hash / "prior <word>" to avoid matching
|
||||
// "verified ... in production" (PR without \b-ish anchor previously
|
||||
// consumed "pr" of "production").
|
||||
/\bverified\s+(?:end[- ]to[- ]end\s+)?(?:on|against|in)\s+(?:PR\s*#?\d+|commit\s+[0-9a-f]{6,}|prior\s+\w+|the\s+\w+\s+audit)\b/i,
|
||||
/\btested\s+(?:against|in|on)\s+(?:PR\s*#?\d+|commit\s+[0-9a-f]{6,}|prior\s+\w+)\b/i,
|
||||
// Direct PR/commit references: "PR #8", "on PR 9", "from commit abc123"
|
||||
/\b(?:on|from|in|via|per)\s+PR\s*#?\d+\b/i,
|
||||
/\b(?:from|in|per|against)\s+commit\s+[0-9a-f]{6,}/i,
|
||||
// Observational descriptions of prior behavior: "was flipping", "was X before", "previously observed"
|
||||
/\b(?:was|were)\s+(?:flipping|drifting|inconsistent|non[- ]deterministic|creeping)\b/i,
|
||||
/\bpreviously\s+(?:observed|flagged|reported|seen|landed)\b/i,
|
||||
/\bused\s+to\s+(?:flip|fail|flag|reject|block)\b/i,
|
||||
/\bobserved\s+(?:in|during|on|across)\s+(?:PR|prior|\d+\s+(?:runs?|audits?))/i,
|
||||
// "flipping/drifting across N runs" — historical variance description
|
||||
/\b(?:flipping|drifting|varying|oscillating)\s+across\s+(?:\d+\s+)?(?:runs?|audits?|iterations?)\b/i,
|
||||
// "the proven X" referring to prior work (proven is a STRONG pattern
|
||||
// but in context "the proven FOO" is usually a historical reference,
|
||||
// not a fresh claim). We catch it here so the empirical skip wins.
|
||||
/\bthe\s+proven\s+(?:escalation\s+ladder|pipeline|flow|loop|tier|path)/i,
|
||||
// "from the 9-run test", "across the 5-run validation"
|
||||
/\b(?:from|across|in|during)\s+the\s+\d+[- ]run\s+(?:test|validation|probe|experiment)/i,
|
||||
];
|
||||
|
||||
export interface ParsedClaims {
|
||||
claims: Claim[];
|
||||
commits_scanned: number;
|
||||
}
|
||||
|
||||
export function parseClaims(pr: PrSnapshot): ParsedClaims {
|
||||
const claims: Claim[] = [];
|
||||
|
||||
// PR body — every matching line becomes a claim at location "pr_body:N"
|
||||
if (pr.body) {
|
||||
scanText(pr.body, "pr_body", pr.head_sha, claims);
|
||||
}
|
||||
|
||||
// Each commit message gets its own scan.
|
||||
for (const c of pr.commits) {
|
||||
if (!c.message) continue;
|
||||
scanText(c.message, `commit:${c.sha.slice(0, 8)}`, c.sha, claims);
|
||||
}
|
||||
|
||||
return { claims, commits_scanned: pr.commits.length };
|
||||
}
|
||||
|
||||
function scanText(text: string, location_prefix: string, commit_sha: string, out: Claim[]): void {
|
||||
const lines = text.split(/\r?\n/);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line.length < 3) continue;
|
||||
|
||||
// Empirical match wins over everything else — if a line ALSO
|
||||
// contains a moderate word like "complete", we still want to
|
||||
// classify it as empirical so the inference check doesn't ask
|
||||
// the cloud to prove "58 cloud calls" from the diff. Order:
|
||||
// empirical → strong → moderate → weak.
|
||||
const empirical = firstUnquotedMatch(line, EMPIRICAL_PATTERNS);
|
||||
if (empirical) {
|
||||
out.push({
|
||||
text: line.trim().slice(0, 200),
|
||||
commit_sha,
|
||||
location: `${location_prefix}:${i + 1}`,
|
||||
strength: "empirical",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const strong = firstUnquotedMatch(line, STRONG_PATTERNS);
|
||||
if (strong) {
|
||||
out.push({
|
||||
text: line.trim().slice(0, 200),
|
||||
commit_sha,
|
||||
location: `${location_prefix}:${i + 1}`,
|
||||
strength: "strong",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const moderate = firstUnquotedMatch(line, MODERATE_PATTERNS);
|
||||
if (moderate) {
|
||||
out.push({
|
||||
text: line.trim().slice(0, 200),
|
||||
commit_sha,
|
||||
location: `${location_prefix}:${i + 1}`,
|
||||
strength: "moderate",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const weak = firstUnquotedMatch(line, WEAK_PATTERNS);
|
||||
if (weak) {
|
||||
out.push({
|
||||
text: line.trim().slice(0, 200),
|
||||
commit_sha,
|
||||
location: `${location_prefix}:${i + 1}`,
|
||||
strength: "weak",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Match a pattern only when its match position is NOT inside a quoted
|
||||
// string on the line. Mirrors the same guard in auditor/checks/static.ts
|
||||
// — the two files have the same false-positive class: PR authors
|
||||
// quote pattern examples in commit message bodies (e.g. `"Phase 45
|
||||
// shipped"` as a test example) and without this guard those quoted
|
||||
// references get flagged as fresh ship-claims. Only skips when the
|
||||
// match itself falls inside quotes; real (unquoted) uses of the same
|
||||
// vocabulary still classify correctly.
|
||||
function firstUnquotedMatch(text: string, patterns: RegExp[]): RegExp | null {
|
||||
for (const p of patterns) {
|
||||
const m = text.match(p);
|
||||
if (!m || typeof m.index !== "number") continue;
|
||||
if (isInsideQuotedString(text, m.index)) continue;
|
||||
return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Walks left→right toggling in-quote state on each unescaped quote.
|
||||
// Good enough for single-line claims; multi-line strings aren't parsed.
|
||||
function isInsideQuotedString(line: string, pos: number): boolean {
|
||||
let inDouble = false, inSingle = false, inBacktick = false;
|
||||
for (let i = 0; i < pos; i++) {
|
||||
const c = line[i];
|
||||
const esc = i > 0 && line[i - 1] === "\\";
|
||||
if (esc) continue;
|
||||
if (c === '"' && !inSingle && !inBacktick) inDouble = !inDouble;
|
||||
else if (c === "'" && !inDouble && !inBacktick) inSingle = !inSingle;
|
||||
else if (c === "`" && !inDouble && !inSingle) inBacktick = !inBacktick;
|
||||
}
|
||||
return inDouble || inSingle || inBacktick;
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
// fact_extractor — routes curated TEXT through llm_team_ui's
|
||||
// "knowledge extract facts" mode (mode=extract at /api/run).
|
||||
//
|
||||
// What it gives us: structured {facts, entities, relationships} from
|
||||
// whatever curated blob we send. Auditor sends the tree-split
|
||||
// inference scratchpad (the best distillation of what a PR changed).
|
||||
// Scrum_master will later send its accepted review bodies.
|
||||
//
|
||||
// Why route through llm_team and not just extract directly from our
|
||||
// own checks: llm_team's extract uses a local EXTRACTOR model
|
||||
// (qwen2.5) + a separate VERIFIER (gemma2). This cross-check is the
|
||||
// discipline J wants for knowledge going into the playbook — facts
|
||||
// go in only after a second model has rated them CORRECT /
|
||||
// UNVERIFIABLE. Fast (local models, ~10-20s), free, and matches the
|
||||
// codereview pattern J already trusts.
|
||||
//
|
||||
// SSE parsing: llm_team streams SSE events. We're only interested in
|
||||
// the final "response" event with role="final" + the extraction
|
||||
// response (role="extraction N"). Parse the JSON from the extractor's
|
||||
// response text.
|
||||
|
||||
const LLM_TEAM = process.env.LH_LLM_TEAM_URL ?? "http://localhost:5000";
|
||||
const EXTRACTOR = process.env.LH_FACT_EXTRACTOR ?? "qwen2.5:latest";
|
||||
const VERIFIER = process.env.LH_FACT_VERIFIER ?? "gemma2:latest";
|
||||
const EXTRACT_TIMEOUT_MS = 120_000;
|
||||
const PROJECT_CONTEXT_FILE = process.env.LH_AUDITOR_CONTEXT_FILE
|
||||
?? "/home/profit/lakehouse/docs/AUDITOR_CONTEXT.md";
|
||||
|
||||
let cachedContext: string | null = null;
|
||||
async function loadProjectContext(): Promise<string> {
|
||||
if (cachedContext !== null) return cachedContext;
|
||||
try {
|
||||
const { readFile } = await import("node:fs/promises");
|
||||
const raw = await readFile(PROJECT_CONTEXT_FILE, "utf8");
|
||||
// Cap at 4KB — anything past that is more noise than signal for
|
||||
// the extractor/verifier's attention budget.
|
||||
cachedContext = raw.slice(0, 4000);
|
||||
} catch {
|
||||
cachedContext = ""; // context file missing → extractor runs without preamble
|
||||
}
|
||||
return cachedContext;
|
||||
}
|
||||
|
||||
export interface Entity {
|
||||
name: string;
|
||||
type: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface Relationship {
|
||||
from: string;
|
||||
to: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface ExtractedFacts {
|
||||
facts: string[];
|
||||
entities: Entity[];
|
||||
relationships: Relationship[];
|
||||
verification: string;
|
||||
extractor_model: string;
|
||||
verifier_model: string;
|
||||
source_preview: string;
|
||||
// Populated when the extract run completed server-side (llm_team
|
||||
// persists to its own team_runs; this is for our own cross-ref).
|
||||
llm_team_run_id?: number;
|
||||
extracted_at: string;
|
||||
// Per-fact verdicts from the verifier pass (CORRECT/INCORRECT/
|
||||
// UNVERIFIABLE/UNCHECKED). Aligned 1:1 with the *raw* fact list
|
||||
// pre-drop so operators can see which verdicts mapped to dropped
|
||||
// facts if needed.
|
||||
verifier_verdicts?: Array<"CORRECT" | "INCORRECT" | "UNVERIFIABLE" | "UNCHECKED">;
|
||||
facts_dropped_by_verifier?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the llm_team extract pipeline on `source` text. Returns
|
||||
* structured {facts, entities, relationships}.
|
||||
*
|
||||
* Returns an object with `error` set if the pipeline failed — never
|
||||
* throws, because fact extraction is best-effort enrichment (the
|
||||
* primary audit must not break if llm_team is down).
|
||||
*/
|
||||
export async function extractFacts(source: string): Promise<ExtractedFacts> {
|
||||
const base: ExtractedFacts = {
|
||||
facts: [],
|
||||
entities: [],
|
||||
relationships: [],
|
||||
verification: "",
|
||||
extractor_model: EXTRACTOR,
|
||||
verifier_model: VERIFIER,
|
||||
source_preview: source.slice(0, 240),
|
||||
extracted_at: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// Prepend project context to the source so the extractor + verifier
|
||||
// know what codebase/framework these facts belong to. Without this,
|
||||
// the verifier marks most domain-specific facts as UNVERIFIABLE ("I
|
||||
// don't know what Lakehouse is"). With it, the verifier can CORRECT-
|
||||
// stamp facts that align with the stated architecture.
|
||||
const context = await loadProjectContext();
|
||||
const prompt = context.length > 0
|
||||
? `=== PROJECT CONTEXT (for grounding facts; do NOT extract facts from this section) ===\n${context}\n\n=== CONTENT TO EXTRACT FACTS FROM ===\n${source}`
|
||||
: source;
|
||||
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(`${LLM_TEAM}/api/run`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
mode: "extract",
|
||||
prompt,
|
||||
extractor: EXTRACTOR,
|
||||
verifier: VERIFIER,
|
||||
source: "prompt",
|
||||
skip_cache: true, // cache by prompt would dedup identical
|
||||
// scratchpads, but we want fresh extraction
|
||||
// for per-audit facts; cheap since local.
|
||||
}),
|
||||
signal: AbortSignal.timeout(EXTRACT_TIMEOUT_MS),
|
||||
});
|
||||
} catch (e) {
|
||||
return { ...base, error: `fetch failed: ${(e as Error).message}` };
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text().catch(() => "");
|
||||
return { ...base, error: `llm_team /api/run ${resp.status}: ${body.slice(0, 200)}` };
|
||||
}
|
||||
|
||||
// Stream SSE lines; collect the one extraction response + the run_saved event
|
||||
// so we can capture the team-runs ID for cross-ref.
|
||||
const decoder = new TextDecoder();
|
||||
const reader = resp.body?.getReader();
|
||||
if (!reader) return { ...base, error: "no response body" };
|
||||
|
||||
let buffer = "";
|
||||
let extractionText = "";
|
||||
let verifierText = "";
|
||||
let runId: number | undefined = undefined;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
let nl: number;
|
||||
while ((nl = buffer.indexOf("\n\n")) >= 0) {
|
||||
const chunk = buffer.slice(0, nl);
|
||||
buffer = buffer.slice(nl + 2);
|
||||
const dataLine = chunk.split("\n").find(l => l.startsWith("data: "));
|
||||
if (!dataLine) continue;
|
||||
try {
|
||||
const ev = JSON.parse(dataLine.slice(6));
|
||||
if (ev.type === "response") {
|
||||
const role = String(ev.role ?? "");
|
||||
if (role.startsWith("extraction")) extractionText = String(ev.text ?? "");
|
||||
else if (role === "verifier") verifierText = String(ev.text ?? "");
|
||||
} else if (ev.type === "run_saved") {
|
||||
const id = Number(ev.run_id);
|
||||
if (Number.isFinite(id)) runId = id;
|
||||
}
|
||||
} catch { /* skip malformed SSE */ }
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
return { ...base, error: `SSE read failed: ${(e as Error).message}` };
|
||||
}
|
||||
|
||||
// Pull the JSON object out of extractionText (may be wrapped in ```json fences).
|
||||
const parsed = extractFirstJsonObject(extractionText);
|
||||
if (!parsed) {
|
||||
return { ...base, error: "extractor returned no parseable JSON", verification: verifierText };
|
||||
}
|
||||
|
||||
const rawFacts: string[] = Array.isArray(parsed.facts)
|
||||
? parsed.facts.slice(0, 50).map(String)
|
||||
: [];
|
||||
|
||||
// Parse the verifier's free-form prose into per-fact verdicts, then
|
||||
// drop any fact the verifier explicitly marked INCORRECT. Leave
|
||||
// UNVERIFIABLE in place: many of our extractions are domain-specific
|
||||
// (Lakehouse internals) and the verifier has no prior-knowledge
|
||||
// anchor, so UNVERIFIABLE is the expected verdict for new signal,
|
||||
// not a quality fail. This is verifier-gated persistence: drop only
|
||||
// what's affirmatively wrong, not what's novel.
|
||||
const verdicts = parseVerifierVerdicts(verifierText, rawFacts.length);
|
||||
const incorrectIdx = new Set<number>();
|
||||
verdicts.forEach((v, i) => { if (v === "INCORRECT") incorrectIdx.add(i); });
|
||||
const kept = rawFacts.filter((_, i) => !incorrectIdx.has(i));
|
||||
|
||||
return {
|
||||
...base,
|
||||
facts: kept,
|
||||
entities: Array.isArray(parsed.entities)
|
||||
? parsed.entities.slice(0, 30).map((e: any) => ({
|
||||
name: String(e?.name ?? ""),
|
||||
type: String(e?.type ?? ""),
|
||||
description: typeof e?.description === "string" ? e.description.slice(0, 240) : undefined,
|
||||
})).filter(e => e.name.length > 0)
|
||||
: [],
|
||||
relationships: Array.isArray(parsed.relationships)
|
||||
? parsed.relationships.slice(0, 30).map((r: any) => ({
|
||||
from: String(r?.from ?? ""),
|
||||
to: String(r?.to ?? ""),
|
||||
type: String(r?.type ?? ""),
|
||||
})).filter(r => r.from.length > 0 && r.to.length > 0)
|
||||
: [],
|
||||
verification: verifierText.slice(0, 1500),
|
||||
facts_dropped_by_verifier: incorrectIdx.size,
|
||||
verifier_verdicts: verdicts,
|
||||
llm_team_run_id: runId,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse verifier's free-form output into a per-fact verdict array.
|
||||
// Gemma2 uses several formats depending on prompt mood:
|
||||
// Format A: **1.** claim... * **Verdict:** CORRECT
|
||||
// Format B: **1.** claim... * **CORRECT** (no "Verdict:" label)
|
||||
// Format C: 1. claim... CORRECT
|
||||
// Strategy: split on fact numbers, then find the first
|
||||
// CORRECT|INCORRECT|UNVERIFIABLE token in each section. Handles all
|
||||
// three formats without regex gymnastics.
|
||||
function parseVerifierVerdicts(
|
||||
verifierText: string,
|
||||
numFacts: number,
|
||||
): Array<"CORRECT" | "INCORRECT" | "UNVERIFIABLE" | "UNCHECKED"> {
|
||||
const out: Array<"CORRECT" | "INCORRECT" | "UNVERIFIABLE" | "UNCHECKED"> =
|
||||
Array(numFacts).fill("UNCHECKED");
|
||||
if (!verifierText) return out;
|
||||
|
||||
// Find each fact section start — "**N.**" or "N." at line start —
|
||||
// and slice out the content up to the NEXT fact number. Each section
|
||||
// gets scanned for the first CORRECT/INCORRECT/UNVERIFIABLE token.
|
||||
const starts: Array<{ idx: number; pos: number }> = [];
|
||||
const header = /(?:^|\n)\s*(?:\*\*)?(\d+)[.)]/g;
|
||||
for (const m of verifierText.matchAll(header)) {
|
||||
const factNum = Number(m[1]);
|
||||
if (!Number.isFinite(factNum)) continue;
|
||||
starts.push({ idx: factNum - 1, pos: m.index! });
|
||||
}
|
||||
for (let i = 0; i < starts.length; i++) {
|
||||
const s = starts[i];
|
||||
const end = i + 1 < starts.length ? starts[i + 1].pos : verifierText.length;
|
||||
if (s.idx < 0 || s.idx >= numFacts) continue;
|
||||
const section = verifierText.slice(s.pos, end);
|
||||
const v = section.match(/\b(CORRECT|INCORRECT|UNVERIFIABLE)\b/i);
|
||||
if (v) out[s.idx] = v[1].toUpperCase() as "CORRECT" | "INCORRECT" | "UNVERIFIABLE";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Lift the first balanced JSON object out of (possibly fenced) text.
|
||||
// Same discipline as inference.ts::extractJson.
|
||||
function extractFirstJsonObject(text: string): any | null {
|
||||
const cleaned = text.replace(/^```(?:json)?\s*/im, "").replace(/```\s*$/im, "");
|
||||
let depth = 0, start = -1;
|
||||
for (let i = 0; i < cleaned.length; i++) {
|
||||
const c = cleaned[i];
|
||||
if (c === "{") { if (depth === 0) start = i; depth++; }
|
||||
else if (c === "}") {
|
||||
depth--;
|
||||
if (depth === 0 && start >= 0) {
|
||||
try { return JSON.parse(cleaned.slice(start, i + 1)); } catch { start = -1; }
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Standalone runner for auditor fixtures. Invoke:
|
||||
//
|
||||
// bun run auditor/fixtures/cli.ts hybrid_38_40_45
|
||||
//
|
||||
// Prints human-readable per-layer breakdown + JSON result so the
|
||||
// output can be captured by the dynamic check without re-running.
|
||||
|
||||
import { runHybridFixture, type FixtureResult } from "./hybrid_38_40_45.ts";
|
||||
|
||||
const fixtureArg = process.argv[2] ?? "hybrid_38_40_45";
|
||||
|
||||
let result: FixtureResult;
|
||||
switch (fixtureArg) {
|
||||
case "hybrid_38_40_45":
|
||||
result = await runHybridFixture();
|
||||
break;
|
||||
default:
|
||||
console.error(`unknown fixture: ${fixtureArg}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// Human-readable summary
|
||||
console.error(""); // blank line to stderr so stdout JSON stays clean
|
||||
console.error(`─── Fixture: ${result.fixture} ───`);
|
||||
console.error(` overall: ${result.overall.toUpperCase()}`);
|
||||
console.error(` shipped: [${result.shipped_phases.join(", ") || "—"}]`);
|
||||
console.error(` placeholder: [${result.placeholder_phases.join(", ") || "—"}]`);
|
||||
console.error("");
|
||||
for (const l of result.layers) {
|
||||
const mark = l.ok ? "✓" : "✗";
|
||||
const phaseStr = `Phase ${l.phase}`.padEnd(11);
|
||||
console.error(` ${mark} ${phaseStr} ${l.layer.padEnd(30)} ${String(l.latency_ms).padStart(6)}ms`);
|
||||
if (l.ok) {
|
||||
console.error(` ${l.evidence.slice(0, 200)}`);
|
||||
} else {
|
||||
console.error(` ERROR: ${l.error?.slice(0, 240) ?? "unknown"}`);
|
||||
}
|
||||
}
|
||||
console.error("");
|
||||
for (const n of result.notes) console.error(` note: ${n}`);
|
||||
console.error("");
|
||||
|
||||
// Machine-readable output on stdout.
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
|
||||
// Exit code reflects overall: 0 pass, 2 partial, 1 fail.
|
||||
// Dynamic check reads this AND the JSON; partial-pass is treated
|
||||
// as informative (some layers shipped), not blocking on its own.
|
||||
process.exit(result.overall === "pass" ? 0 : result.overall === "partial_pass" ? 2 : 1);
|
||||
@@ -0,0 +1,335 @@
|
||||
// The never-run hybrid test fixture. Exercises the full stack end-to-end
|
||||
// across Phase 38 (/v1/chat), Phase 40 (Langfuse tracing), Phase 45
|
||||
// slice 1 (DocRef on playbook), and Phase 45 slice 2 (context7 bridge
|
||||
// drift detection).
|
||||
//
|
||||
// Returns HONEST per-layer results. A layer that's unimplemented or
|
||||
// broken returns ok=false with a real error; overall verdict flags
|
||||
// which phases are genuinely shipped vs still placeholder.
|
||||
//
|
||||
// Deterministic: always runs the same payload so a run's output is
|
||||
// comparable across audits. Uses TEST-prefixed names so playbook
|
||||
// state doesn't get polluted with production-looking fixture data.
|
||||
//
|
||||
// Preconditions:
|
||||
// - gateway up on :3100
|
||||
// - Python sidecar up on :3200
|
||||
// - Ollama local + Ollama Cloud key loaded
|
||||
// - Langfuse up on :3001
|
||||
// - context7 bridge up on :3900 (manual-start per mcp-server/README)
|
||||
// If bridge isn't running, that layer reports ok=false with a
|
||||
// clear error — the fixture doesn't try to auto-start it.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
const GATEWAY = process.env.LH_GATEWAY_URL ?? "http://localhost:3100";
|
||||
const LANGFUSE = process.env.LANGFUSE_URL ?? "http://localhost:3001";
|
||||
const BRIDGE = process.env.CONTEXT7_BRIDGE_URL ?? "http://localhost:3900";
|
||||
const FIXTURE_ID = "auditor:hybrid_38_40_45:v1";
|
||||
const TEST_WORKER_NAME = "__auditor_test_worker__";
|
||||
const STALE_HASH = "stale-hash-for-drift-proof";
|
||||
// Unique-per-run identifiers so each fixture invocation hits the ADD
|
||||
// path in upsert_entry (not UPDATE/NOOP on a leftover from a prior
|
||||
// run). Catches state pollution + avoids interaction with task #12
|
||||
// (UPDATE branch silently drops doc_refs).
|
||||
const RUN_NONCE = Date.now().toString(36);
|
||||
const TEST_WORKER_NAME_VERSIONED = `__auditor_test_worker_${RUN_NONCE}__`;
|
||||
const TEST_OPERATION_VERSIONED = `TEST: fill: __auditor_${RUN_NONCE}__ x1 in TestCity, TC`;
|
||||
|
||||
export interface LayerResult {
|
||||
layer: string;
|
||||
phase: string;
|
||||
ok: boolean;
|
||||
latency_ms: number;
|
||||
evidence: string;
|
||||
error?: string;
|
||||
// Layer-specific numbers that downstream checks can aggregate.
|
||||
metrics?: Record<string, number | string | boolean>;
|
||||
}
|
||||
|
||||
export interface FixtureResult {
|
||||
fixture: string;
|
||||
ran_at: string;
|
||||
overall: "pass" | "partial_pass" | "fail";
|
||||
layers: LayerResult[];
|
||||
real_numbers: Record<string, number>;
|
||||
shipped_phases: string[];
|
||||
placeholder_phases: string[];
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
// Wraps a layer's async body with timing + structured error capture.
|
||||
// On throw: returns ok=false with the error string as evidence, so
|
||||
// downstream verdict code treats it as a layer failure rather than
|
||||
// a fixture crash.
|
||||
async function measureLayer(
|
||||
layer: string,
|
||||
phase: string,
|
||||
body: () => Promise<{ evidence: string; metrics?: Record<string, number | string | boolean> }>,
|
||||
): Promise<LayerResult> {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const { evidence, metrics } = await body();
|
||||
return {
|
||||
layer, phase, ok: true,
|
||||
latency_ms: Date.now() - t0,
|
||||
evidence, metrics,
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
layer, phase, ok: false,
|
||||
latency_ms: Date.now() - t0,
|
||||
evidence: `FAILED: ${(e as Error).message.slice(0, 200)}`,
|
||||
error: (e as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function langfuseAuth(): Promise<{ pk: string; sk: string } | null> {
|
||||
// Default credentials match mcp-server/tracing.ts. Same sources the
|
||||
// Rust side uses in gateway/src/v1/langfuse_trace.rs.
|
||||
let pk = process.env.LANGFUSE_PUBLIC_KEY ?? "pk-lf-staffing";
|
||||
let sk = process.env.LANGFUSE_SECRET_KEY ?? "sk-lf-staffing-secret";
|
||||
if (!pk || !sk) return null;
|
||||
return { pk, sk };
|
||||
}
|
||||
|
||||
export async function runHybridFixture(): Promise<FixtureResult> {
|
||||
const result: FixtureResult = {
|
||||
fixture: FIXTURE_ID,
|
||||
ran_at: new Date().toISOString(),
|
||||
overall: "fail",
|
||||
layers: [],
|
||||
real_numbers: {},
|
||||
shipped_phases: [],
|
||||
placeholder_phases: [],
|
||||
notes: [],
|
||||
};
|
||||
|
||||
// ========================================================================
|
||||
// Layer 1 — Phase 38: POST /v1/chat returns valid OpenAI shape
|
||||
// ========================================================================
|
||||
// Captured HERE, immediately before the chat layer runs, so layer 2's
|
||||
// Langfuse-trace filter uses the actual moment the chat call was
|
||||
// attempted — not the fixture start time. Earlier draft had a
|
||||
// meaningless ternary returning result.ran_at on both branches; the
|
||||
// LLM-Team codereview (2026-04-22) caught this and flagged it as a
|
||||
// false-negative window on traces created between fixture-start and
|
||||
// chat-fetch.
|
||||
const chat_request_sent_ms = Date.now();
|
||||
|
||||
const l1 = await measureLayer("phase38_chat", "38", async () => {
|
||||
const r = await fetch(`${GATEWAY}/v1/chat`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "qwen3.5:latest",
|
||||
messages: [
|
||||
{ role: "system", content: "Respond in 8 words or fewer." },
|
||||
{ role: "user", content: "Name one Docker command." },
|
||||
],
|
||||
max_tokens: 60,
|
||||
temperature: 0.2,
|
||||
}),
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
if (!r.ok) throw new Error(`gateway ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
const content = j.choices?.[0]?.message?.content;
|
||||
if (typeof content !== "string" || content.length === 0) {
|
||||
throw new Error(`empty content — think-false default may not be wired: ${JSON.stringify(j).slice(0, 200)}`);
|
||||
}
|
||||
return {
|
||||
evidence: `content="${content.slice(0, 60)}" tokens=${j.usage?.total_tokens}`,
|
||||
metrics: {
|
||||
prompt_tokens: j.usage?.prompt_tokens ?? 0,
|
||||
completion_tokens: j.usage?.completion_tokens ?? 0,
|
||||
total_tokens: j.usage?.total_tokens ?? 0,
|
||||
content_nonempty: true,
|
||||
},
|
||||
};
|
||||
});
|
||||
result.layers.push(l1);
|
||||
|
||||
// ========================================================================
|
||||
// Layer 2 — Phase 40: Langfuse trace lands within 3s
|
||||
// ========================================================================
|
||||
// Fire-and-forget tracing means we need a brief sleep before query.
|
||||
await new Promise(res => setTimeout(res, 2500));
|
||||
const l2 = await measureLayer("phase40_langfuse_trace", "40", async () => {
|
||||
const auth = await langfuseAuth();
|
||||
if (!auth) throw new Error("Langfuse credentials not in env");
|
||||
const r = await fetch(`${LANGFUSE}/api/public/traces?limit=5&name=v1.chat:ollama`, {
|
||||
headers: { Authorization: `Basic ${btoa(`${auth.pk}:${auth.sk}`)}` },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!r.ok) throw new Error(`langfuse ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
const items = Array.isArray(j.data) ? j.data : [];
|
||||
// Filter on the chat-request timestamp captured above. A Langfuse
|
||||
// trace must be newer than the moment we fired /v1/chat to plausibly
|
||||
// belong to our request. Using fixture start time (result.ran_at)
|
||||
// was wrong and could false-negative on slow fixtures.
|
||||
const recent = items.filter((t: any) => Date.parse(t.timestamp) >= chat_request_sent_ms);
|
||||
if (recent.length === 0) {
|
||||
throw new Error(`no v1.chat:ollama trace since ${new Date(chat_request_sent_ms).toISOString()} (${items.length} older traces visible, Langfuse reachable — tracing is not firing)`);
|
||||
}
|
||||
const trace = recent[0];
|
||||
return {
|
||||
evidence: `trace ${trace.id.slice(0, 8)} name=${trace.name} age=${Date.now() - Date.parse(trace.timestamp)}ms`,
|
||||
metrics: {
|
||||
trace_age_ms: Date.now() - Date.parse(trace.timestamp),
|
||||
trace_latency_reported: Number(trace.latency ?? 0),
|
||||
recent_trace_count: recent.length,
|
||||
},
|
||||
};
|
||||
});
|
||||
result.layers.push(l2);
|
||||
|
||||
// ========================================================================
|
||||
// Layer 3 — Phase 45 slice 1: /seed accepts doc_refs and persists them
|
||||
// ========================================================================
|
||||
// Seed a playbook with doc_refs referencing Docker at a stale hash.
|
||||
// Ignore the cleanup concern for v1 — this fixture pollutes the
|
||||
// in-memory playbook state; operator can retire test entries.
|
||||
const l3 = await measureLayer("phase45_seed_with_doc_refs", "45.1", async () => {
|
||||
const r = await fetch(`${GATEWAY}/vectors/playbook_memory/seed`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
operation: TEST_OPERATION_VERSIONED,
|
||||
approach: "auditor hybrid fixture run",
|
||||
context: "doc_refs integration test — retire after audit",
|
||||
endorsed_names: [TEST_WORKER_NAME_VERSIONED],
|
||||
append: true,
|
||||
doc_refs: [
|
||||
{
|
||||
tool: "docker",
|
||||
version_seen: "24.0.7",
|
||||
snippet_hash: STALE_HASH,
|
||||
source_url: "https://context7.com/docker/docs",
|
||||
seen_at: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!r.ok) throw new Error(`seed ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
// Verify doc_refs actually persisted — read state.json directly.
|
||||
// That's the canonical source of truth; the seed response may or
|
||||
// may not echo doc_refs back.
|
||||
const state_raw = await readFile("/home/profit/lakehouse/data/_playbook_memory/state.json", "utf8");
|
||||
const state = JSON.parse(state_raw);
|
||||
const entries = state.entries ?? [];
|
||||
const ours = entries.find((e: any) => (e.endorsed_names ?? []).includes(TEST_WORKER_NAME_VERSIONED));
|
||||
if (!ours) throw new Error(`seeded entry not found in state.json (worker ${TEST_WORKER_NAME_VERSIONED} not present)`);
|
||||
const docRefs = ours.doc_refs ?? [];
|
||||
if (docRefs.length === 0) {
|
||||
throw new Error("entry saved but doc_refs field is empty — the field accepted by the API isn't being persisted");
|
||||
}
|
||||
return {
|
||||
evidence: `playbook ${ours.playbook_id.slice(0, 16)} persisted with doc_refs.length=${docRefs.length}; first tool=${docRefs[0]?.tool} hash=${docRefs[0]?.snippet_hash}`,
|
||||
metrics: {
|
||||
doc_refs_stored: docRefs.length,
|
||||
entries_after: j.entries_after ?? -1,
|
||||
playbook_id: ours.playbook_id,
|
||||
mode: j.outcome?.mode ?? "unknown",
|
||||
},
|
||||
};
|
||||
});
|
||||
result.layers.push(l3);
|
||||
|
||||
// ========================================================================
|
||||
// Layer 4 — Phase 45 slice 2: context7 bridge detects drift vs stale hash
|
||||
// ========================================================================
|
||||
const l4 = await measureLayer("phase45_bridge_diff", "45.2", async () => {
|
||||
// Health check first — if bridge isn't running, fail with a clear
|
||||
// message rather than mysterious connection refused.
|
||||
try {
|
||||
const h = await fetch(`${BRIDGE}/health`, { signal: AbortSignal.timeout(3000) });
|
||||
if (!h.ok) throw new Error(`bridge /health ${h.status}`);
|
||||
} catch (e) {
|
||||
throw new Error(`context7 bridge at ${BRIDGE} is not reachable (bridge is manual-start; run 'bun run mcp-server/context7_bridge.ts'): ${(e as Error).message}`);
|
||||
}
|
||||
const r = await fetch(`${BRIDGE}/docs/docker/diff?since=${encodeURIComponent(STALE_HASH)}`, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!r.ok) throw new Error(`bridge diff ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
if (j.drifted !== true) {
|
||||
throw new Error(`expected drifted=true against ${STALE_HASH}; got ${JSON.stringify(j)}`);
|
||||
}
|
||||
return {
|
||||
evidence: `bridge confirms drift: current=${j.current_snippet_hash} previous=${j.previous_snippet_hash} upstream_updated=${j.last_updated_upstream}`,
|
||||
metrics: {
|
||||
drifted: true,
|
||||
current_hash_length: String(j.current_snippet_hash ?? "").length,
|
||||
library_id: j.library_id,
|
||||
},
|
||||
};
|
||||
});
|
||||
result.layers.push(l4);
|
||||
|
||||
// ========================================================================
|
||||
// Layer 5 — Phase 45 slice 3: /doc_drift/check/{id} endpoint
|
||||
// (EXPECTED TO FAIL — endpoint unimplemented)
|
||||
// ========================================================================
|
||||
const l5 = await measureLayer("phase45_slice3_drift_flag", "45.3", async () => {
|
||||
const pid = String(l3.metrics?.playbook_id ?? "");
|
||||
if (!pid) throw new Error("layer 3 did not produce a playbook_id to check");
|
||||
const r = await fetch(`${GATEWAY}/vectors/playbook_memory/doc_drift/check/${encodeURIComponent(pid)}`, {
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (r.status === 404 || r.status === 405) {
|
||||
// This is the HONEST signal: the endpoint doesn't exist yet.
|
||||
// Fail this layer with a clear marker — not a silent pass.
|
||||
throw new Error(`endpoint unimplemented (${r.status}) — Phase 45 slice 3 is still placeholder. doc_refs are persisted (layer 3) and the bridge can answer drift questions (layer 4), but nothing ties them together into a playbook flag yet.`);
|
||||
}
|
||||
if (!r.ok) throw new Error(`drift/check ${r.status}: ${await r.text()}`);
|
||||
const j: any = await r.json();
|
||||
// If the endpoint DID exist, assert the flag got set.
|
||||
if (!j.flagged) {
|
||||
throw new Error(`endpoint responded but flag not set: ${JSON.stringify(j).slice(0, 200)}`);
|
||||
}
|
||||
return {
|
||||
evidence: `playbook flagged: ${JSON.stringify(j).slice(0, 160)}`,
|
||||
metrics: { flagged: true },
|
||||
};
|
||||
});
|
||||
result.layers.push(l5);
|
||||
|
||||
// ========================================================================
|
||||
// Verdict assembly
|
||||
// ========================================================================
|
||||
for (const l of result.layers) {
|
||||
if (l.ok) result.shipped_phases.push(l.phase);
|
||||
else result.placeholder_phases.push(l.phase);
|
||||
result.real_numbers[`${l.layer}_latency_ms`] = l.latency_ms;
|
||||
if (l.metrics) {
|
||||
for (const [k, v] of Object.entries(l.metrics)) {
|
||||
if (typeof v === "number") result.real_numbers[`${l.layer}.${k}`] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const anyFail = result.placeholder_phases.length > 0;
|
||||
const allPass = result.layers.every(l => l.ok);
|
||||
result.overall = allPass ? "pass" : (result.shipped_phases.length > 0 ? "partial_pass" : "fail");
|
||||
|
||||
if (!l5.ok) {
|
||||
result.notes.push(
|
||||
"Phase 45 slice 3 (doc_drift/check endpoint) is the expected-fail layer. " +
|
||||
"Layer 5 failing with a 404/405 is the CORRECT honest signal — don't mask this " +
|
||||
"to get a green result. When slice 3 ships, layer 5 flips to ok=true automatically."
|
||||
);
|
||||
}
|
||||
if (anyFail) {
|
||||
result.notes.push(
|
||||
`${result.placeholder_phases.length} layer(s) failed. Check evidence per layer for specifics.`
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// Gitea API client. Minimal surface — only what the auditor needs:
|
||||
// list open PRs, get commits + files for a PR, fetch a diff, post a
|
||||
// commit status, post a review.
|
||||
//
|
||||
// Auth: reads PAT from ~/.git-credentials (set up by the credential
|
||||
// helper flow in 2026-04-22 session). Gitea's "token" auth scheme
|
||||
// matches what `git fetch` is already using.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { PrSnapshot } from "./types.ts";
|
||||
|
||||
const HOST = process.env.GITEA_HOST ?? "https://git.agentview.dev";
|
||||
const OWNER = "profit";
|
||||
const REPO = "lakehouse";
|
||||
const CRED_FILE = "/home/profit/.git-credentials";
|
||||
|
||||
let cachedPat: string | null = null;
|
||||
|
||||
async function getPat(): Promise<string> {
|
||||
if (cachedPat) return cachedPat;
|
||||
const raw = await readFile(CRED_FILE, "utf8");
|
||||
for (const line of raw.split("\n")) {
|
||||
const m = line.match(/^https:\/\/[^:]+:([^@]+)@git\.agentview\.dev/);
|
||||
if (m) { cachedPat = m[1]; return m[1]; }
|
||||
}
|
||||
throw new Error(`no Gitea PAT in ${CRED_FILE}`);
|
||||
}
|
||||
|
||||
async function giteaFetch(path: string, init: RequestInit = {}): Promise<Response> {
|
||||
const pat = await getPat();
|
||||
const url = `${HOST}/api/v1${path}`;
|
||||
const headers = new Headers(init.headers);
|
||||
headers.set("Authorization", `token ${pat}`);
|
||||
if (init.body && !headers.has("content-type")) {
|
||||
headers.set("content-type", "application/json");
|
||||
}
|
||||
return fetch(url, { ...init, headers, signal: AbortSignal.timeout(20000) });
|
||||
}
|
||||
|
||||
export async function listOpenPrs(): Promise<PrSnapshot[]> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls?state=open&page=1&limit=50`);
|
||||
if (!r.ok) throw new Error(`listOpenPrs ${r.status}: ${await r.text()}`);
|
||||
const rows = (await r.json()) as any[];
|
||||
return Promise.all(rows.map(row => snapshotFromPr(row)));
|
||||
}
|
||||
|
||||
export async function getPrSnapshot(num: number): Promise<PrSnapshot> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}`);
|
||||
if (!r.ok) throw new Error(`getPr ${num} ${r.status}: ${await r.text()}`);
|
||||
return snapshotFromPr((await r.json()) as any);
|
||||
}
|
||||
|
||||
async function snapshotFromPr(row: any): Promise<PrSnapshot> {
|
||||
const num = row.number;
|
||||
const commitsResp = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}/commits`);
|
||||
const commits = commitsResp.ok ? ((await commitsResp.json()) as any[]) : [];
|
||||
const filesResp = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}/files`);
|
||||
const files = filesResp.ok ? ((await filesResp.json()) as any[]) : [];
|
||||
return {
|
||||
number: num,
|
||||
head_sha: row.head?.sha ?? "",
|
||||
base_sha: row.base?.sha ?? "",
|
||||
title: row.title ?? "",
|
||||
body: row.body ?? "",
|
||||
state: row.state === "open" ? "open" : (row.merged ? "merged" : "closed"),
|
||||
author: row.user?.login ?? "",
|
||||
commits: commits.map(c => ({
|
||||
sha: (c.sha ?? "").slice(0, 12),
|
||||
message: c.commit?.message ?? "",
|
||||
author: c.commit?.author?.name ?? "",
|
||||
})),
|
||||
files: files.map(f => ({
|
||||
path: f.filename ?? "",
|
||||
additions: f.additions ?? 0,
|
||||
deletions: f.deletions ?? 0,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the unified diff text of the PR. Used by static checks.
|
||||
export async function getPrDiff(num: number): Promise<string> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}.diff`);
|
||||
if (!r.ok) throw new Error(`getDiff ${num} ${r.status}: ${await r.text()}`);
|
||||
return await r.text();
|
||||
}
|
||||
|
||||
/// Hard-block mechanism: post a failing commit status on the PR head
|
||||
/// SHA. Branch protection (if enabled on `main`) treats this as a
|
||||
/// required-check fail and prevents merge. The description is shown
|
||||
/// in the Gitea UI next to the red X.
|
||||
export async function postCommitStatus(args: {
|
||||
sha: string;
|
||||
state: "success" | "pending" | "failure" | "error";
|
||||
context: string;
|
||||
description: string;
|
||||
target_url?: string;
|
||||
}): Promise<void> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/statuses/${args.sha}`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
state: args.state,
|
||||
context: args.context,
|
||||
description: args.description.slice(0, 140),
|
||||
target_url: args.target_url ?? "",
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`postCommitStatus ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
|
||||
/// Post a review comment. Gitea typically blocks self-review
|
||||
/// (author posting a review on their own PR). Prefer
|
||||
/// `postIssueComment` when running with the author's PAT.
|
||||
export async function postReview(args: {
|
||||
pr_number: number;
|
||||
commit_id: string;
|
||||
body: string;
|
||||
event: "APPROVE" | "REQUEST_CHANGES" | "COMMENT";
|
||||
}): Promise<void> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${args.pr_number}/reviews`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
commit_id: args.commit_id,
|
||||
body: args.body,
|
||||
event: args.event,
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`postReview ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
|
||||
/// Plain issue comment. Works for the auditor's own PAT because
|
||||
/// Gitea allows authors to comment on their own PRs (just not
|
||||
/// review them). Auditor uses this for the reasoning body; the
|
||||
/// actual block signal is the commit status.
|
||||
export async function postIssueComment(args: {
|
||||
pr_number: number;
|
||||
body: string;
|
||||
}): Promise<{ id: number; html_url: string }> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/issues/${args.pr_number}/comments`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ body: args.body }),
|
||||
});
|
||||
if (!r.ok) throw new Error(`postIssueComment ${r.status}: ${await r.text()}`);
|
||||
const j = await r.json() as any;
|
||||
return { id: j.id, html_url: j.html_url };
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// Auditor poller — the top-level entry. Polls Gitea for open PRs on
|
||||
// a fixed interval, dedupes by head SHA, runs audit + posts verdict
|
||||
// for each new (pr, sha) pair.
|
||||
//
|
||||
// Run manually:
|
||||
// bun run auditor/index.ts
|
||||
//
|
||||
// Stop:
|
||||
// touch auditor.paused (skips next cycle)
|
||||
// pkill -f auditor/index.ts (kills in-flight)
|
||||
//
|
||||
// State:
|
||||
// data/_auditor/state.json — last-audited SHA per PR
|
||||
// data/_auditor/verdicts/{id}.json — per-run verdict records
|
||||
//
|
||||
// This entry runs forever. A systemd unit would wrap it once the
|
||||
// workflow is trusted (same pattern as mcp-server, observer).
|
||||
|
||||
import { readFile, writeFile, mkdir, access } from "node:fs/promises";
|
||||
import { listOpenPrs } from "./gitea.ts";
|
||||
import { auditPr } from "./audit.ts";
|
||||
|
||||
const POLL_INTERVAL_MS = 90_000; // 90s — enough budget for audit runs to complete
|
||||
const PAUSE_FILE = "/home/profit/lakehouse/auditor.paused";
|
||||
const STATE_FILE = "/home/profit/lakehouse/data/_auditor/state.json";
|
||||
|
||||
interface State {
|
||||
// Map: PR number → last-audited head SHA. Lets us dedupe audits
|
||||
// across restarts (poller can crash/restart without re-auditing
|
||||
// all open PRs from scratch).
|
||||
last_audited: Record<string, string>;
|
||||
started_at: string;
|
||||
cycles_total: number;
|
||||
cycles_skipped_paused: number;
|
||||
audits_run: number;
|
||||
last_cycle_at?: string;
|
||||
}
|
||||
|
||||
async function fileExists(path: string): Promise<boolean> {
|
||||
try { await access(path); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function loadState(): Promise<State> {
|
||||
try {
|
||||
const raw = await readFile(STATE_FILE, "utf8");
|
||||
const s = JSON.parse(raw);
|
||||
return {
|
||||
last_audited: s.last_audited ?? {},
|
||||
started_at: s.started_at ?? new Date().toISOString(),
|
||||
cycles_total: s.cycles_total ?? 0,
|
||||
cycles_skipped_paused: s.cycles_skipped_paused ?? 0,
|
||||
audits_run: s.audits_run ?? 0,
|
||||
last_cycle_at: s.last_cycle_at,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
last_audited: {},
|
||||
started_at: new Date().toISOString(),
|
||||
cycles_total: 0,
|
||||
cycles_skipped_paused: 0,
|
||||
audits_run: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function saveState(s: State): Promise<void> {
|
||||
await mkdir("/home/profit/lakehouse/data/_auditor", { recursive: true });
|
||||
await writeFile(STATE_FILE, JSON.stringify(s, null, 2));
|
||||
}
|
||||
|
||||
async function runCycle(state: State): Promise<State> {
|
||||
state.cycles_total += 1;
|
||||
state.last_cycle_at = new Date().toISOString();
|
||||
|
||||
if (await fileExists(PAUSE_FILE)) {
|
||||
state.cycles_skipped_paused += 1;
|
||||
console.log(`[auditor] cycle ${state.cycles_total}: paused (touch ${PAUSE_FILE} exists)`);
|
||||
return state;
|
||||
}
|
||||
|
||||
let prs;
|
||||
try {
|
||||
prs = await listOpenPrs();
|
||||
} catch (e) {
|
||||
console.error(`[auditor] listOpenPrs failed: ${(e as Error).message}`);
|
||||
return state;
|
||||
}
|
||||
|
||||
console.log(`[auditor] cycle ${state.cycles_total}: ${prs.length} open PR(s)`);
|
||||
|
||||
for (const pr of prs) {
|
||||
const last = state.last_audited[String(pr.number)];
|
||||
if (last === pr.head_sha) {
|
||||
console.log(`[auditor] skip PR #${pr.number} (SHA ${pr.head_sha.slice(0, 8)} already audited)`);
|
||||
continue;
|
||||
}
|
||||
console.log(`[auditor] audit PR #${pr.number} (${pr.head_sha.slice(0, 8)}) — ${pr.title.slice(0, 60)}`);
|
||||
try {
|
||||
// Skip dynamic by default: it mutates live playbook state and
|
||||
// re-runs on every PR update would pollute quickly. Operator
|
||||
// can run dynamic via `bun run auditor/fixtures/cli.ts` manually
|
||||
// OR set LH_AUDITOR_RUN_DYNAMIC=1 to opt in.
|
||||
const run_dynamic = process.env.LH_AUDITOR_RUN_DYNAMIC === "1";
|
||||
const verdict = await auditPr(pr, {
|
||||
skip_dynamic: !run_dynamic,
|
||||
skip_inference: process.env.LH_AUDITOR_SKIP_INFERENCE === "1",
|
||||
});
|
||||
console.log(`[auditor] verdict=${verdict.overall} findings=${verdict.metrics.findings_total} (block=${verdict.metrics.findings_block} warn=${verdict.metrics.findings_warn})`);
|
||||
state.last_audited[String(pr.number)] = pr.head_sha;
|
||||
state.audits_run += 1;
|
||||
} catch (e) {
|
||||
console.error(`[auditor] audit failed: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`[auditor] starting poller — interval ${POLL_INTERVAL_MS / 1000}s`);
|
||||
console.log(`[auditor] pause file: ${PAUSE_FILE}`);
|
||||
console.log(`[auditor] state file: ${STATE_FILE}`);
|
||||
|
||||
let state = await loadState();
|
||||
console.log(`[auditor] loaded state: ${Object.keys(state.last_audited).length} PRs previously audited, ${state.cycles_total} cycles so far`);
|
||||
|
||||
// Single-shot mode for CLI testing: `bun run auditor/index.ts --once`
|
||||
const once = process.argv.includes("--once");
|
||||
if (once) {
|
||||
state = await runCycle(state);
|
||||
await saveState(state);
|
||||
console.log(`[auditor] single-shot complete. total audits: ${state.audits_run}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Loop.
|
||||
while (true) {
|
||||
state = await runCycle(state);
|
||||
await saveState(state);
|
||||
await new Promise(res => setTimeout(res, POLL_INTERVAL_MS));
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error("[auditor] fatal:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
// kb_index — generic on-the-fly aggregation over append-only JSONL
|
||||
// scratchpads (audit_lessons, scrum_reviews, outcomes, observer ops).
|
||||
//
|
||||
// The mem0 insight: raw rows are CHEAP and tell the full story, but
|
||||
// downstream prompts need a DEFINITION, not a log. A definition is
|
||||
// the aggregate: "this signature has fired N times across M distinct
|
||||
// scopes, first_seen=X, last_seen=Y, confidence=M/N."
|
||||
//
|
||||
// This library is the single shared aggregator. Every KB writer keeps
|
||||
// appending raw rows; every KB reader uses aggregate() instead of
|
||||
// tailing the raw stream. No second file to sync, no ADD/UPDATE/NOOP
|
||||
// routing — the stats roll up from the raw rows every time.
|
||||
//
|
||||
// Why this works past hundreds of runs:
|
||||
// - aggregate() is bounded by distinct_signatures, not total_rows.
|
||||
// - confidence = distinct_scopes / count — low for same-scope noise,
|
||||
// high for cross-scope patterns. Downstream severity ramps on
|
||||
// confidence × count, not raw count, so one unfixed PR can't
|
||||
// inflate its own recurrence score (the classic mem0 failure).
|
||||
// - rotation (later) moves old raw to archive files; aggregate()
|
||||
// can still read both to compute lifetime counts when needed.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
export interface AggregateRow {
|
||||
signature: string;
|
||||
count: number;
|
||||
distinct_scopes: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
confidence: number; // distinct_scopes / count — capped at 1.0
|
||||
representative_summary: string; // most-recent summary for this signature
|
||||
scopes: string[]; // up to 20 most-recent scopes for debugging
|
||||
checks: string[]; // distinct `check` values (audit_lessons-specific)
|
||||
}
|
||||
|
||||
export interface AggregateOptions<T> {
|
||||
/** How to extract the dedup key from a row. */
|
||||
keyFn: (row: T) => string | undefined;
|
||||
/** How to extract the "scope" — distinct scopes count gives confidence. */
|
||||
scopeFn: (row: T) => string | undefined;
|
||||
/** How to extract the timestamp (defaults to row.audited_at / row.reviewed_at / row.timestamp). */
|
||||
timeFn?: (row: T) => string | undefined;
|
||||
/** How to extract a representative summary (defaults to row.summary). */
|
||||
summaryFn?: (row: T) => string | undefined;
|
||||
/** Max rows to read from the JSONL tail; 0 = read all. */
|
||||
tailLimit?: number;
|
||||
/** Include per-row check field (for multi-check aggregates). */
|
||||
checkFn?: (row: T) => string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a JSONL file and produce the aggregate map keyed by signature.
|
||||
* Safe on missing or malformed files — returns empty map.
|
||||
*/
|
||||
export async function aggregate<T = any>(
|
||||
jsonlPath: string,
|
||||
opts: AggregateOptions<T>,
|
||||
): Promise<Map<string, AggregateRow>> {
|
||||
const out = new Map<string, AggregateRow>();
|
||||
let raw: string;
|
||||
try { raw = await readFile(jsonlPath, "utf8"); } catch { return out; }
|
||||
const lines = raw.split("\n").filter(l => l.length > 0);
|
||||
const sliceFrom = opts.tailLimit && opts.tailLimit > 0 ? Math.max(0, lines.length - opts.tailLimit) : 0;
|
||||
|
||||
const timeFn = opts.timeFn ?? ((r: any) => r?.audited_at ?? r?.reviewed_at ?? r?.timestamp ?? r?.ran_at);
|
||||
const summaryFn = opts.summaryFn ?? ((r: any) => r?.summary ?? r?.representative_summary);
|
||||
|
||||
// Per-signature scope tracking — need counts by scope to compute
|
||||
// distinct_scopes without double-counting a scope that appears 50
|
||||
// times. Using a Set<scope> per signature.
|
||||
const scopeSets = new Map<string, Set<string>>();
|
||||
const checkSets = new Map<string, Set<string>>();
|
||||
|
||||
for (let i = sliceFrom; i < lines.length; i++) {
|
||||
let row: T;
|
||||
try { row = JSON.parse(lines[i]) as T; } catch { continue; }
|
||||
const sig = opts.keyFn(row);
|
||||
if (!sig) continue;
|
||||
|
||||
let agg = out.get(sig);
|
||||
if (!agg) {
|
||||
agg = {
|
||||
signature: sig,
|
||||
count: 0,
|
||||
distinct_scopes: 0,
|
||||
first_seen: "",
|
||||
last_seen: "",
|
||||
confidence: 0,
|
||||
representative_summary: "",
|
||||
scopes: [],
|
||||
checks: [],
|
||||
};
|
||||
out.set(sig, agg);
|
||||
scopeSets.set(sig, new Set<string>());
|
||||
checkSets.set(sig, new Set<string>());
|
||||
}
|
||||
|
||||
agg.count += 1;
|
||||
|
||||
const scope = opts.scopeFn(row);
|
||||
if (scope !== undefined && scope !== null && scope !== "") {
|
||||
scopeSets.get(sig)!.add(String(scope));
|
||||
// Keep scopes array ordered by recency (newest wins — shift
|
||||
// oldest when at cap).
|
||||
const arr = agg.scopes;
|
||||
const s = String(scope);
|
||||
const existing = arr.indexOf(s);
|
||||
if (existing >= 0) arr.splice(existing, 1);
|
||||
arr.push(s);
|
||||
if (arr.length > 20) arr.shift();
|
||||
}
|
||||
|
||||
if (opts.checkFn) {
|
||||
const c = opts.checkFn(row);
|
||||
if (c) checkSets.get(sig)!.add(String(c));
|
||||
}
|
||||
|
||||
const t = timeFn(row);
|
||||
if (t) {
|
||||
if (!agg.first_seen || t < agg.first_seen) agg.first_seen = t;
|
||||
if (!agg.last_seen || t > agg.last_seen) agg.last_seen = t;
|
||||
}
|
||||
|
||||
const s = summaryFn(row);
|
||||
if (s) agg.representative_summary = String(s);
|
||||
}
|
||||
|
||||
// Finalize derived fields.
|
||||
for (const [sig, agg] of out) {
|
||||
const scopes = scopeSets.get(sig) ?? new Set<string>();
|
||||
agg.distinct_scopes = scopes.size;
|
||||
agg.confidence = agg.count > 0 ? Math.min(1, agg.distinct_scopes / agg.count) : 0;
|
||||
const checks = checkSets.get(sig);
|
||||
if (checks) agg.checks = Array.from(checks).sort();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Severity policy derived from aggregate stats. The rating lives here
|
||||
* (not in each check) so all KB readers ramp severity consistently.
|
||||
*
|
||||
* - confidence × count product is the real signal.
|
||||
* - Low confidence (< 0.3) = same-scope noise → info regardless of count.
|
||||
* - Mid confidence (0.3-0.6) = mixed signal → warn at count ≥ 3.
|
||||
* - High confidence (> 0.6) with count ≥ 5 = block-worthy cross-cutting pattern.
|
||||
*
|
||||
* Callers can override by reading agg directly; this is the default
|
||||
* policy that matches the "don't escalate one unfixed PR" discipline.
|
||||
*/
|
||||
export function ratingSeverity(agg: AggregateRow): "info" | "warn" | "block" {
|
||||
if (agg.confidence >= 0.6 && agg.count >= 5) return "block";
|
||||
if (agg.confidence >= 0.3 && agg.count >= 3) return "warn";
|
||||
return "info";
|
||||
}
|
||||
|
||||
/** Human-friendly one-line summary of an aggregate row for finding evidence. */
|
||||
export function formatAgg(agg: AggregateRow): string {
|
||||
return `count=${agg.count} distinct_scopes=${agg.distinct_scopes} confidence=${agg.confidence.toFixed(2)} seen=[${agg.first_seen.slice(0, 10)}..${agg.last_seen.slice(0, 10)}]`;
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// kb_stats — on-demand dashboard numbers from the KB scratchpad
|
||||
// files. Reads data/_auditor/verdicts/*, data/_kb/audit_lessons.jsonl,
|
||||
// data/_kb/audit_facts.jsonl, data/_kb/audit_discrepancies.jsonl,
|
||||
// data/_kb/scrum_reviews.jsonl and prints:
|
||||
//
|
||||
// - verdict flip-flop rate (same SHA re-audited, verdict changed?)
|
||||
// - consensus discrepancy rate (N runs disagreed on a claim)
|
||||
// - confidence distribution from kb_index aggregator
|
||||
// - top N recurring entities from audit_facts
|
||||
// - fact growth over time
|
||||
// - scrum vs inference KB split
|
||||
//
|
||||
// Run: bun run auditor/kb_stats.ts
|
||||
// bun run auditor/kb_stats.ts --top 15 # show top 15 entities
|
||||
// bun run auditor/kb_stats.ts --json # machine-readable
|
||||
//
|
||||
// This is the "dashboard" without running Grafana. If someone really
|
||||
// wants a dashboard, wire this output into a static HTML page + cron.
|
||||
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { aggregate } from "./kb_index.ts";
|
||||
|
||||
const REPO = "/home/profit/lakehouse";
|
||||
const VERDICTS_DIR = `${REPO}/data/_auditor/verdicts`;
|
||||
const AUDIT_LESSONS = `${REPO}/data/_kb/audit_lessons.jsonl`;
|
||||
const AUDIT_FACTS = `${REPO}/data/_kb/audit_facts.jsonl`;
|
||||
const AUDIT_DISCREPANCIES = `${REPO}/data/_kb/audit_discrepancies.jsonl`;
|
||||
const SCRUM_REVIEWS = `${REPO}/data/_kb/scrum_reviews.jsonl`;
|
||||
|
||||
interface Args {
|
||||
top: number;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
const a: Args = { top: 10, json: false };
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--top") a.top = Number(argv[++i] ?? 10);
|
||||
else if (argv[i] === "--json") a.json = true;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
async function readJsonl<T = any>(path: string): Promise<T[]> {
|
||||
try {
|
||||
const raw = await readFile(path, "utf8");
|
||||
return raw.split("\n").filter(l => l.length > 0).map(l => {
|
||||
try { return JSON.parse(l) as T; } catch { return null as any; }
|
||||
}).filter(r => r !== null);
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
async function loadVerdicts(): Promise<Array<{ pr: number; sha: string; overall: string; findings_total: number; findings_block: number; findings_warn: number }>> {
|
||||
let files: string[] = [];
|
||||
try { files = await readdir(VERDICTS_DIR); } catch { return []; }
|
||||
const out = [];
|
||||
for (const f of files) {
|
||||
if (!f.endsWith(".json")) continue;
|
||||
const m = f.match(/^(\d+)-([0-9a-f]+)\.json$/);
|
||||
if (!m) continue;
|
||||
try {
|
||||
const v = JSON.parse(await readFile(join(VERDICTS_DIR, f), "utf8"));
|
||||
out.push({
|
||||
pr: Number(m[1]),
|
||||
sha: m[2],
|
||||
overall: String(v.overall),
|
||||
findings_total: Number(v.metrics?.findings_total ?? 0),
|
||||
findings_block: Number(v.metrics?.findings_block ?? 0),
|
||||
findings_warn: Number(v.metrics?.findings_warn ?? 0),
|
||||
});
|
||||
} catch { /* skip corrupt */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface Stats {
|
||||
audit_count: number;
|
||||
verdict_distribution: Record<string, number>;
|
||||
// Same PR with multiple SHAs — if verdicts differ, that's drift across
|
||||
// the PR's commit history. Not a flip-flop in the classical sense,
|
||||
// but worth surfacing (e.g. "PR #8 was block block req req block").
|
||||
per_pr_verdict_sequences: Record<number, string[]>;
|
||||
// For each PR with ≥ 2 audits, how many distinct verdicts did it
|
||||
// produce? 1 = stable; 2+ = some flipping.
|
||||
verdict_instability: { pr_count: number; pr_with_multiple_verdicts: number; pr_with_3plus: number };
|
||||
consensus: { discrepancy_count: number; tiebreaker_used: number; unresolved: number };
|
||||
kb: {
|
||||
audit_lessons_rows: number;
|
||||
audit_facts_rows: number;
|
||||
scrum_reviews_rows: number;
|
||||
distinct_finding_signatures: number;
|
||||
distinct_entities_across_prs: number;
|
||||
entities_in_2plus_prs: number;
|
||||
entities_in_5plus_prs: number;
|
||||
};
|
||||
fact_quality: {
|
||||
verifier_verdict_distribution: Record<string, number>;
|
||||
facts_dropped_by_verifier_total: number;
|
||||
extraction_success_rate: number;
|
||||
};
|
||||
top_entities: Array<{ name: string; distinct_prs: number; count: number; types: string[] }>;
|
||||
kb_by_source: Record<string, number>;
|
||||
}
|
||||
|
||||
async function collect(args: Args): Promise<Stats> {
|
||||
const verdicts = await loadVerdicts();
|
||||
const lessons = await readJsonl<any>(AUDIT_LESSONS);
|
||||
const facts = await readJsonl<any>(AUDIT_FACTS);
|
||||
const disc = await readJsonl<any>(AUDIT_DISCREPANCIES);
|
||||
const reviews = await readJsonl<any>(SCRUM_REVIEWS);
|
||||
|
||||
// Verdict stability
|
||||
const byPr: Record<number, string[]> = {};
|
||||
const verdictDist: Record<string, number> = {};
|
||||
for (const v of verdicts) {
|
||||
(byPr[v.pr] ??= []).push(v.overall);
|
||||
verdictDist[v.overall] = (verdictDist[v.overall] ?? 0) + 1;
|
||||
}
|
||||
let multi = 0, tri = 0;
|
||||
for (const [_, seq] of Object.entries(byPr)) {
|
||||
const distinct = new Set(seq);
|
||||
if (distinct.size >= 2) multi++;
|
||||
if (distinct.size >= 3) tri++;
|
||||
}
|
||||
|
||||
// Consensus drift
|
||||
const consensus = {
|
||||
discrepancy_count: disc.length,
|
||||
tiebreaker_used: disc.filter(d => String(d.resolution).startsWith("tiebreaker")).length,
|
||||
unresolved: disc.filter(d => d.resolution === "unresolved").length,
|
||||
};
|
||||
|
||||
// Lesson signatures
|
||||
const lessonAgg = await aggregate<any>(AUDIT_LESSONS, {
|
||||
keyFn: r => r?.signature,
|
||||
scopeFn: r => (r?.pr_number !== undefined ? `pr-${r.pr_number}` : undefined),
|
||||
});
|
||||
|
||||
// Entity aggregation across audit_facts rows
|
||||
interface EntAgg { distinct_prs: Set<number>; count: number; types: Set<string>; name: string; sources: Set<string> }
|
||||
const entAgg = new Map<string, EntAgg>();
|
||||
const sourceCount: Record<string, number> = {};
|
||||
let totalVerdictDist: Record<string, number> = { CORRECT: 0, INCORRECT: 0, UNVERIFIABLE: 0, UNCHECKED: 0 };
|
||||
let factsDroppedTotal = 0;
|
||||
let extractionsWithFacts = 0;
|
||||
|
||||
for (const row of facts) {
|
||||
const src = String(row.source ?? "unknown");
|
||||
sourceCount[src] = (sourceCount[src] ?? 0) + 1;
|
||||
const pr = Number(row.pr_number);
|
||||
if (Array.isArray(row.verifier_verdicts)) {
|
||||
for (const v of row.verifier_verdicts) {
|
||||
totalVerdictDist[v] = (totalVerdictDist[v] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
factsDroppedTotal += Number(row.facts_dropped_by_verifier ?? 0);
|
||||
if ((Array.isArray(row.facts) && row.facts.length > 0) || (Array.isArray(row.entities) && row.entities.length > 0)) {
|
||||
extractionsWithFacts++;
|
||||
}
|
||||
for (const e of Array.isArray(row.entities) ? row.entities : []) {
|
||||
const name = String(e?.name ?? "").trim();
|
||||
if (name.length < 3) continue;
|
||||
const key = name.toLowerCase();
|
||||
const agg = entAgg.get(key) ?? { distinct_prs: new Set(), count: 0, types: new Set(), name, sources: new Set() };
|
||||
agg.count++;
|
||||
if (Number.isFinite(pr) && pr > 0) agg.distinct_prs.add(pr);
|
||||
if (e?.type) agg.types.add(String(e.type));
|
||||
agg.sources.add(src);
|
||||
entAgg.set(key, agg);
|
||||
}
|
||||
}
|
||||
|
||||
const entitiesIn2Plus = Array.from(entAgg.values()).filter(a => a.distinct_prs.size >= 2).length;
|
||||
const entitiesIn5Plus = Array.from(entAgg.values()).filter(a => a.distinct_prs.size >= 5).length;
|
||||
const topEntities = Array.from(entAgg.values())
|
||||
.sort((a, b) => b.distinct_prs.size - a.distinct_prs.size || b.count - a.count)
|
||||
.slice(0, args.top)
|
||||
.map(a => ({
|
||||
name: a.name,
|
||||
distinct_prs: a.distinct_prs.size,
|
||||
count: a.count,
|
||||
types: Array.from(a.types),
|
||||
}));
|
||||
|
||||
const stats: Stats = {
|
||||
audit_count: verdicts.length,
|
||||
verdict_distribution: verdictDist,
|
||||
per_pr_verdict_sequences: byPr,
|
||||
verdict_instability: {
|
||||
pr_count: Object.keys(byPr).length,
|
||||
pr_with_multiple_verdicts: multi,
|
||||
pr_with_3plus: tri,
|
||||
},
|
||||
consensus,
|
||||
kb: {
|
||||
audit_lessons_rows: lessons.length,
|
||||
audit_facts_rows: facts.length,
|
||||
scrum_reviews_rows: reviews.length,
|
||||
distinct_finding_signatures: lessonAgg.size,
|
||||
distinct_entities_across_prs: entAgg.size,
|
||||
entities_in_2plus_prs: entitiesIn2Plus,
|
||||
entities_in_5plus_prs: entitiesIn5Plus,
|
||||
},
|
||||
fact_quality: {
|
||||
verifier_verdict_distribution: totalVerdictDist,
|
||||
facts_dropped_by_verifier_total: factsDroppedTotal,
|
||||
extraction_success_rate: facts.length > 0 ? extractionsWithFacts / facts.length : 0,
|
||||
},
|
||||
top_entities: topEntities,
|
||||
kb_by_source: sourceCount,
|
||||
};
|
||||
return stats;
|
||||
}
|
||||
|
||||
function renderHuman(s: Stats): string {
|
||||
const lines: string[] = [];
|
||||
lines.push("═══ KB STATS ═══");
|
||||
lines.push("");
|
||||
lines.push(`Audits: ${s.audit_count} total across ${s.verdict_instability.pr_count} distinct PRs`);
|
||||
lines.push(`Verdicts: ${Object.entries(s.verdict_distribution).map(([k, v]) => `${k}=${v}`).join(" ")}`);
|
||||
const multiplePct = s.verdict_instability.pr_count > 0
|
||||
? Math.round(100 * s.verdict_instability.pr_with_multiple_verdicts / s.verdict_instability.pr_count)
|
||||
: 0;
|
||||
lines.push(`Verdict instability: ${s.verdict_instability.pr_with_multiple_verdicts}/${s.verdict_instability.pr_count} PRs had 2+ distinct verdicts (${multiplePct}%) — 3+ distinct: ${s.verdict_instability.pr_with_3plus}`);
|
||||
lines.push("");
|
||||
lines.push("─── Consensus ───");
|
||||
lines.push(` discrepancies logged: ${s.consensus.discrepancy_count}`);
|
||||
lines.push(` tiebreaker used: ${s.consensus.tiebreaker_used}`);
|
||||
lines.push(` unresolved: ${s.consensus.unresolved}`);
|
||||
const dRate = s.audit_count > 0 ? (100 * s.consensus.discrepancy_count / s.audit_count).toFixed(1) : "0";
|
||||
lines.push(` discrepancy rate: ${dRate}% of audits`);
|
||||
lines.push("");
|
||||
lines.push("─── KB size ───");
|
||||
lines.push(` audit_lessons.jsonl: ${s.kb.audit_lessons_rows} rows, ${s.kb.distinct_finding_signatures} distinct signatures`);
|
||||
lines.push(` audit_facts.jsonl: ${s.kb.audit_facts_rows} rows, ${s.kb.distinct_entities_across_prs} distinct entities`);
|
||||
lines.push(` scrum_reviews.jsonl: ${s.kb.scrum_reviews_rows} rows`);
|
||||
lines.push(` entities in 2+ PRs: ${s.kb.entities_in_2plus_prs}`);
|
||||
lines.push(` entities in 5+ PRs: ${s.kb.entities_in_5plus_prs} ← strong cross-cutting signal`);
|
||||
lines.push("");
|
||||
lines.push("─── Fact quality ───");
|
||||
const v = s.fact_quality.verifier_verdict_distribution;
|
||||
lines.push(` verifier verdicts: CORRECT=${v.CORRECT ?? 0} UNVERIFIABLE=${v.UNVERIFIABLE ?? 0} UNCHECKED=${v.UNCHECKED ?? 0} INCORRECT=${v.INCORRECT ?? 0}`);
|
||||
lines.push(` facts dropped by verifier: ${s.fact_quality.facts_dropped_by_verifier_total}`);
|
||||
lines.push(` extraction success rate: ${(s.fact_quality.extraction_success_rate * 100).toFixed(1)}%`);
|
||||
lines.push("");
|
||||
lines.push("─── KB sources ───");
|
||||
for (const [src, n] of Object.entries(s.kb_by_source)) {
|
||||
lines.push(` ${src}: ${n}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push(`─── Top ${s.top_entities.length} recurring entities ───`);
|
||||
for (const e of s.top_entities) {
|
||||
lines.push(` [${e.distinct_prs} PRs × ${e.count} obs] ${e.name} (${e.types.join(",")})`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
const stats = await collect(args);
|
||||
if (args.json) {
|
||||
console.log(JSON.stringify(stats, (_, v) => v instanceof Set ? Array.from(v) : v, 2));
|
||||
} else {
|
||||
console.log(renderHuman(stats));
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => { console.error("[kb_stats] fatal:", e); process.exit(1); });
|
||||
@@ -0,0 +1,62 @@
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// YOU WRITE THIS FILE. Policy decides what blocks vs what's a comment.
|
||||
// Defaults are opinionated on the "stop clicking past placeholder"
|
||||
// side — easier to loosen than to tighten when you're watching the
|
||||
// auditor behave in live PRs.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
import type { Finding, Verdict } from "./types.ts";
|
||||
|
||||
/// Translate the four-check output into a single verdict. This is the
|
||||
/// single pane of glass the auditor operates on — tune thresholds here.
|
||||
export function assembleVerdict(
|
||||
findings: Finding[],
|
||||
metrics: Record<string, number>,
|
||||
pr_number: number,
|
||||
head_sha: string,
|
||||
): Verdict {
|
||||
const blocking = findings.filter(f => f.severity === "block");
|
||||
const warning = findings.filter(f => f.severity === "warn");
|
||||
|
||||
let overall: Verdict["overall"];
|
||||
let one_liner: string;
|
||||
|
||||
if (blocking.length > 0) {
|
||||
overall = "block";
|
||||
one_liner = `${blocking.length} blocking issue${blocking.length > 1 ? "s" : ""}: ${blocking[0].summary}`;
|
||||
} else if (warning.length >= 3) {
|
||||
// Three or more warnings is a block — death by a thousand cuts.
|
||||
overall = "request_changes";
|
||||
one_liner = `${warning.length} warnings — see review`;
|
||||
} else if (warning.length > 0) {
|
||||
overall = "request_changes";
|
||||
one_liner = warning[0].summary;
|
||||
} else {
|
||||
overall = "approve";
|
||||
one_liner = `all checks passed (${findings.length} findings, all info)`;
|
||||
}
|
||||
|
||||
return {
|
||||
pr_number,
|
||||
head_sha,
|
||||
audited_at: new Date().toISOString(),
|
||||
overall,
|
||||
findings,
|
||||
metrics,
|
||||
one_liner,
|
||||
};
|
||||
}
|
||||
|
||||
/// Which strength-of-claim warrants which severity when evidence is
|
||||
/// weak? A "Phase X shipped" claim with zero integration tests is a
|
||||
/// blocker. A "should work" claim with no test is a warn.
|
||||
export function severityFromClaimEvidence(
|
||||
claim_strength: "weak" | "moderate" | "strong",
|
||||
evidence_grade: "none" | "partial" | "full",
|
||||
): "info" | "warn" | "block" {
|
||||
if (evidence_grade === "full") return "info";
|
||||
if (claim_strength === "strong" && evidence_grade === "none") return "block";
|
||||
if (claim_strength === "strong" && evidence_grade === "partial") return "warn";
|
||||
if (claim_strength === "moderate" && evidence_grade === "none") return "warn";
|
||||
return "info";
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Shared types for the claim-auditor. Every field exists for a reason;
|
||||
// if something can't be verified from a check, it goes into `evidence`
|
||||
// so the verdict is inspectable, not a black box.
|
||||
|
||||
export type CheckKind = "static" | "dynamic" | "inference" | "kb_query";
|
||||
|
||||
export type Severity = "info" | "warn" | "block";
|
||||
|
||||
export interface Claim {
|
||||
// Verbatim phrase that raised the claim — e.g. "Phase 38 shipped",
|
||||
// "verified end-to-end", "works after restart". Used as the "what
|
||||
// does the author assert" input to downstream checks.
|
||||
text: string;
|
||||
// Where it came from. `commit_sha` is the short hash; `location`
|
||||
// is a file:line for in-diff claims, or "pr_body" / "commit_message".
|
||||
commit_sha: string;
|
||||
location: string;
|
||||
// Heuristic rating of how strong the claim is. "green+tested"
|
||||
// is strong; "should work" is weak. Drives sensitivity — stronger
|
||||
// claims get harder-blocked on weak evidence.
|
||||
//
|
||||
// "empirical" is a separate class: runtime measurements like
|
||||
// "N cloud calls" / "306s end-to-end" / "accepted on attempt N".
|
||||
// These cannot be verified from a static diff — only from the test
|
||||
// output that produced them. Inference skips diff-verification for
|
||||
// empirical claims; they become info-level context unless a future
|
||||
// runtime_evidence check contradicts them.
|
||||
strength: "weak" | "moderate" | "strong" | "empirical";
|
||||
}
|
||||
|
||||
export interface Finding {
|
||||
check: CheckKind;
|
||||
severity: Severity;
|
||||
claim_text?: string;
|
||||
// Free-form short description: "field added but never read", "no
|
||||
// test covers this code path", "cloud model says placeholder".
|
||||
summary: string;
|
||||
// Concrete evidence: file paths, line numbers, log excerpts, test
|
||||
// output, cloud-model verdict. No handwaving.
|
||||
evidence: string[];
|
||||
}
|
||||
|
||||
export interface Verdict {
|
||||
pr_number: number;
|
||||
head_sha: string;
|
||||
audited_at: string;
|
||||
overall: "approve" | "request_changes" | "block";
|
||||
findings: Finding[];
|
||||
// Real numbers that downstream policy can gate on. e.g. if the
|
||||
// hybrid test produced latency numbers or token counts, they
|
||||
// surface here so /auditor/history is queryable.
|
||||
metrics: Record<string, number>;
|
||||
// Short one-line justification for the `overall` verdict. What
|
||||
// gets posted as the commit-status description in Gitea (max 140
|
||||
// chars) must fit here.
|
||||
one_liner: string;
|
||||
}
|
||||
|
||||
export interface PrSnapshot {
|
||||
number: number;
|
||||
head_sha: string;
|
||||
base_sha: string;
|
||||
title: string;
|
||||
body: string;
|
||||
state: "open" | "closed" | "merged";
|
||||
author: string;
|
||||
// Array of commit messages in the PR (not diffs — those are
|
||||
// fetched on-demand per-check).
|
||||
commits: Array<{ sha: string; message: string; author: string }>;
|
||||
// File paths touched by the PR, with lines-added / lines-removed.
|
||||
files: Array<{ path: string; additions: number; deletions: number }>;
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
# Lakehouse PR-Bot
|
||||
|
||||
A local sub-agent that reads the PRD, asks a cloud model for a small change
|
||||
proposal, applies it, runs tests, and opens a draft PR on Gitea. Manual-only
|
||||
right now — no systemd, no cron. Run one cycle at a time and watch.
|
||||
|
||||
## Run one cycle
|
||||
|
||||
```bash
|
||||
cd /home/profit/lakehouse
|
||||
bun run bot/cycle.ts
|
||||
```
|
||||
|
||||
Prerequisites:
|
||||
- Working tree must be clean (`git status` shows nothing). The bot refuses on a dirty tree so its changes don't mix with in-flight work.
|
||||
- Sidecar running on :3200, gateway on :3100, observer on :3800.
|
||||
- At least one PRD line tagged `[bot-eligible]` (see below).
|
||||
- Gitea PAT configured in `~/.git-credentials` (already set up).
|
||||
|
||||
## Tag a gap as bot-eligible
|
||||
|
||||
Add `[bot-eligible]` to any PRD line the bot is allowed to work on:
|
||||
|
||||
```md
|
||||
- [ ] Add a unit test for parse_city_state covering "South Bend, IN" edge case [bot-eligible]
|
||||
```
|
||||
|
||||
The bot scans `docs/PRD.md` for these tags. Each tagged line becomes a candidate.
|
||||
Start small — one tag at a time — until you trust the loop.
|
||||
|
||||
## Stop it mid-cycle
|
||||
|
||||
Create the pause file:
|
||||
|
||||
```bash
|
||||
touch /home/profit/lakehouse/bot.paused
|
||||
```
|
||||
|
||||
The next `bot/cycle.ts` invocation exits immediately with `skipped_pause`.
|
||||
(It does NOT kill a cycle already in-flight — use `Ctrl-C` or `pkill -f bot/cycle.ts` for that.)
|
||||
|
||||
## Budget
|
||||
|
||||
- **20 cloud calls/day, 160k tokens/day** (hard ceiling, see `bot/cost.ts`).
|
||||
- Tracked in `data/_bot/cost-YYYY-MM-DD.json`.
|
||||
- Resets at UTC midnight.
|
||||
|
||||
## Cycle outcomes
|
||||
|
||||
Every run writes a result to `data/_bot/cycles/{cycle_id}.json` and POSTs an event to the observer on :3800.
|
||||
Possible outcomes:
|
||||
|
||||
| Outcome | Meaning |
|
||||
|---|---|
|
||||
| `ok` | PR opened |
|
||||
| `cycle_noop` | proposal applied, every file was **identical to current content** (Mem0 NOOP); no test run, no PR |
|
||||
| `skipped_pause` | pause file present |
|
||||
| `skipped_cost` | daily budget exhausted |
|
||||
| `skipped_policy` | `policy.shouldRunCycle` said no |
|
||||
| `skipped_dirty_tree` | uncommitted changes present |
|
||||
| `skipped_no_gap` | no `[bot-eligible]` tags in PRD |
|
||||
| `model_failed` | sidecar/cloud model errored or returned unparseable JSON |
|
||||
| `proposal_rejected` | `policy.scoreProposal` rejected it (size, path, etc.) |
|
||||
| `apply_failed` | file write errored |
|
||||
| `tests_failed` | cargo or bun test red |
|
||||
| `pr_skipped_by_policy` | green tests, but `policy.shouldOpenPR` said no |
|
||||
| `pr_failed` | Gitea API or git push errored |
|
||||
|
||||
## Mem0-aligned apply semantics
|
||||
|
||||
`apply.ts` categorizes every proposed file into one of three modes:
|
||||
|
||||
| Mode | Trigger | Action |
|
||||
|---|---|---|
|
||||
| **ADD** | `is_new: true` and file doesn't exist | Create the file |
|
||||
| **UPDATE** | `is_new: false`, file exists, content **differs** from current | Overwrite |
|
||||
| **NOOP** | `is_new: false`, file exists, content **matches** current exactly | Skip (no write, no diff) |
|
||||
|
||||
If every file in the proposal is NOOP, the cycle short-circuits to `cycle_noop` **before** running tests or opening a PR. Mismatched shapes (`is_new:true` but file exists, or `is_new:false` but file missing) become `apply_failed` — model state confusion is surfaced, not papered over.
|
||||
|
||||
PR bodies report the three counts separately so reviewers see what actually changed vs. what was confirmed identical.
|
||||
|
||||
## How the bot compounds over cycles
|
||||
|
||||
Every finished cycle persists a `CycleResult` to `data/_bot/cycles/{cycle_id}.json`. At the **start** of the next cycle, `bot/kb.ts::loadHistory(gap_id)` scans that directory, filters to prior cycles on the same gap, and returns the five most recent outcomes (`ok`, `tests_failed`, `proposal_rejected`, `cycle_noop`, etc.) with their reasons and touched files.
|
||||
|
||||
Those outcomes are summarized into a compact block and **injected into the cloud prompt** before asking for a new proposal. The model sees things like:
|
||||
|
||||
```
|
||||
Prior attempts on this gap (3 most recent):
|
||||
- 2026-04-22 03:15 UTC — tests_failed
|
||||
reason: cargo test -p vectord::lance failed on field_type_coerce
|
||||
- 2026-04-22 02:48 UTC — proposal_rejected
|
||||
reason: touched forbidden path (docs/ADR-): docs/ADR-019-update.md
|
||||
- 2026-04-22 01:23 UTC — ok PR: https://git.agentview.dev/profit/lakehouse/pulls/142
|
||||
reason: PR #142 opened
|
||||
|
||||
Learn from these: build on what worked, avoid paths that failed.
|
||||
```
|
||||
|
||||
This is the same compounding pattern `scenario.ts` uses via `kb.loadRecommendation` — the bot's cycles are the bot's memory. No embedding, no separate jsonl, no cross-run orchestration required. First cycle on a new gap skips the block cleanly.
|
||||
|
||||
The observer events POSTed on every cycle carry the same data, so `GET :3800/stats?source=bot` aggregates "% of bot cycles on similar gaps that landed a PR" without extra plumbing.
|
||||
|
||||
## Where YOU edit
|
||||
|
||||
`bot/policy.ts` — four small functions that define what the bot does. The rest
|
||||
(`propose.ts`, `apply.ts`, `test.ts`, `pr.ts`, `cycle.ts`, `kb.ts`) is mechanical
|
||||
orchestration — you shouldn't need to touch it unless a pipeline changes.
|
||||
|
||||
One policy upgrade the history opens up: in `shouldRunCycle`, you can now bail
|
||||
out if the same gap has failed N times in a row. Example addition:
|
||||
|
||||
```ts
|
||||
import { loadHistory, statsFor } from "./kb.ts";
|
||||
// inside shouldRunCycle — but you'd need the gap, which is picked later.
|
||||
// A more natural place is scoreProposal: if prior N failures on this
|
||||
// gap's path+summary pattern, reject before testing.
|
||||
```
|
||||
|
||||
## Hard-coded guardrails (not in policy — can't be disabled)
|
||||
|
||||
- Bot **never deletes files** (`apply.ts` has no delete path).
|
||||
- Bot **never touches** `.git/`, `secrets`, `lakehouse.toml`, `docs/ADR-*`, `docs/DECISIONS.md`, `docs/PRD.md`, `/etc/`, `/root/`, `Cargo.lock`.
|
||||
- All paths validated for traversal (`path/../`) and repo-escape.
|
||||
- PRs always open as **draft** — never auto-merge.
|
||||
- Budget check is before the policy function runs — no way to override the daily cap from `policy.ts`.
|
||||
@@ -0,0 +1,86 @@
|
||||
// Apply a proposal to disk with Mem0-aligned ADD / UPDATE / NOOP
|
||||
// semantics. Whole-file writes only — never deletes. All paths
|
||||
// validated repo-relative + path-traversal-free.
|
||||
//
|
||||
// Three outcomes per file:
|
||||
// ADD — file didn't exist, new content written (is_new must be true)
|
||||
// UPDATE — file existed, content differs from current, overwritten
|
||||
// NOOP — file existed, content identical to current, nothing written
|
||||
//
|
||||
// The NOOP case is important: a cloud proposal that "re-derives" an
|
||||
// existing file shouldn't churn the repo or burn test cycles.
|
||||
|
||||
import { writeFile, readFile, mkdir, access } from "node:fs/promises";
|
||||
import { dirname, resolve, relative } from "node:path";
|
||||
import type { ApplyOutcome, Proposal } from "./types.ts";
|
||||
|
||||
const REPO_ROOT = "/home/profit/lakehouse";
|
||||
|
||||
export async function applyProposal(p: Proposal): Promise<ApplyOutcome> {
|
||||
const added: string[] = [];
|
||||
const updated: string[] = [];
|
||||
const noop: string[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const f of p.files) {
|
||||
const err = validatePath(f.path);
|
||||
if (err) {
|
||||
errors.push(`${f.path}: ${err}`);
|
||||
continue;
|
||||
}
|
||||
const abs = resolve(REPO_ROOT, f.path);
|
||||
try {
|
||||
const exists = await fileExists(abs);
|
||||
|
||||
if (f.is_new && exists) {
|
||||
errors.push(`${f.path}: is_new=true but file exists`);
|
||||
continue;
|
||||
}
|
||||
if (!f.is_new && !exists) {
|
||||
// Proposal claims to update a file that doesn't exist.
|
||||
// Refuse rather than silently ADD — model is confused about
|
||||
// repo state, and we want that surfaced, not papered over.
|
||||
errors.push(`${f.path}: is_new=false but file does not exist`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!f.is_new) {
|
||||
// UPDATE candidate — compare to detect NOOP.
|
||||
const current = await readFile(abs, "utf8");
|
||||
if (current === f.content) {
|
||||
noop.push(f.path);
|
||||
continue;
|
||||
}
|
||||
await writeFile(abs, f.content);
|
||||
updated.push(f.path);
|
||||
} else {
|
||||
// ADD — file doesn't exist yet.
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
await writeFile(abs, f.content);
|
||||
added.push(f.path);
|
||||
}
|
||||
} catch (e) {
|
||||
errors.push(`${f.path}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
return { added, updated, noop, errors };
|
||||
}
|
||||
|
||||
function validatePath(p: string): string | null {
|
||||
if (!p) return "empty path";
|
||||
if (p.startsWith("/")) return "must be repo-relative";
|
||||
if (p.startsWith("..") || p.includes("/../")) return "path traversal";
|
||||
const abs = resolve(REPO_ROOT, p);
|
||||
const rel = relative(REPO_ROOT, abs);
|
||||
if (rel.startsWith("..")) return "resolves outside repo";
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fileExists(abs: string): Promise<boolean> {
|
||||
try {
|
||||
await access(abs);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
// Daily cost tracker. Resets at UTC midnight by keying the file by date.
|
||||
// Hard ceilings are the policy surface — the bot refuses to call the
|
||||
// cloud once the budget is exhausted for the day.
|
||||
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { CostState } from "./types.ts";
|
||||
|
||||
const COST_DIR = "/home/profit/lakehouse/data/_bot";
|
||||
|
||||
export const DAILY_CALLS_BUDGET = 20;
|
||||
export const DAILY_TOKENS_BUDGET = 8000 * DAILY_CALLS_BUDGET; // 160k tokens/day ceiling
|
||||
|
||||
function todayUtc(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function costFile(date: string): string {
|
||||
return join(COST_DIR, `cost-${date}.json`);
|
||||
}
|
||||
|
||||
export async function readCost(): Promise<CostState> {
|
||||
const date = todayUtc();
|
||||
try {
|
||||
const raw = await readFile(costFile(date), "utf8");
|
||||
return JSON.parse(raw) as CostState;
|
||||
} catch {
|
||||
return { date, calls: 0, tokens: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
export async function recordCost(calls: number, tokens: number): Promise<CostState> {
|
||||
await mkdir(COST_DIR, { recursive: true });
|
||||
const current = await readCost();
|
||||
const updated: CostState = {
|
||||
date: current.date,
|
||||
calls: current.calls + calls,
|
||||
tokens: current.tokens + tokens,
|
||||
};
|
||||
await writeFile(costFile(updated.date), JSON.stringify(updated, null, 2));
|
||||
return updated;
|
||||
}
|
||||
|
||||
export function budgetCheck(c: CostState): { ok: boolean; reason: string } {
|
||||
if (c.calls >= DAILY_CALLS_BUDGET) {
|
||||
return { ok: false, reason: `daily call budget exhausted (${c.calls}/${DAILY_CALLS_BUDGET})` };
|
||||
}
|
||||
if (c.tokens >= DAILY_TOKENS_BUDGET) {
|
||||
return { ok: false, reason: `daily token budget exhausted (${c.tokens}/${DAILY_TOKENS_BUDGET})` };
|
||||
}
|
||||
return { ok: true, reason: `${c.calls}/${DAILY_CALLS_BUDGET} calls, ${c.tokens}/${DAILY_TOKENS_BUDGET} tokens used today` };
|
||||
}
|
||||
+345
@@ -0,0 +1,345 @@
|
||||
// Single-cycle orchestrator. This is the manual entry point:
|
||||
// bun run bot/cycle.ts
|
||||
//
|
||||
// Steps (each step can short-circuit with a clear outcome):
|
||||
// 1. Pause-file check → skipped_pause
|
||||
// 2. Cost budget check → skipped_cost
|
||||
// 3. Gather CycleContext (dirty tree, last cycle, autotune status)
|
||||
// 4. policy.shouldRunCycle → skipped_policy
|
||||
// 5. findGaps + policy.pickGap → skipped_no_gap
|
||||
// 6. generateProposal → model_failed
|
||||
// 7. policy.scoreProposal → proposal_rejected
|
||||
// 8. git checkout new branch
|
||||
// 9. applyProposal → apply_failed
|
||||
// 10. runTests → tests_failed
|
||||
// 11. policy.shouldOpenPR → pr_skipped_by_policy
|
||||
// 12. git commit + push
|
||||
// 13. openPr → pr_failed / ok
|
||||
// 14. recordCost + postEvent (every outcome)
|
||||
// 15. write cycle result to data/_bot/cycles/{cycle_id}.json
|
||||
|
||||
import { readFile, writeFile, mkdir, access } from "node:fs/promises";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
|
||||
import type { CycleContext, CycleResult, CycleOutcome, Proposal } from "./types.ts";
|
||||
import * as policy from "./policy.ts";
|
||||
import { readCost, recordCost, budgetCheck, DAILY_CALLS_BUDGET, DAILY_TOKENS_BUDGET } from "./cost.ts";
|
||||
import { findGaps, generateProposal } from "./propose.ts";
|
||||
import { applyProposal } from "./apply.ts";
|
||||
import { runTests } from "./test.ts";
|
||||
import { openPr } from "./pr.ts";
|
||||
import { postEvent } from "./observer.ts";
|
||||
import { loadHistory, summarizeHistory, statsFor } from "./kb.ts";
|
||||
|
||||
const REPO_ROOT = "/home/profit/lakehouse";
|
||||
const PAUSE_FILE = `${REPO_ROOT}/bot.paused`;
|
||||
const CYCLES_DIR = `${REPO_ROOT}/data/_bot/cycles`;
|
||||
const GATEWAY_URL = process.env.LH_GATEWAY_URL ?? "http://localhost:3100";
|
||||
|
||||
function log(msg: string) { console.log(`[bot] ${msg}`); }
|
||||
|
||||
async function fileExists(p: string): Promise<boolean> {
|
||||
try { await access(p); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
function git(args: string[]): { code: number; stdout: string; stderr: string } {
|
||||
const r = spawnSync("git", args, { cwd: REPO_ROOT, encoding: "utf8" });
|
||||
return { code: r.status ?? -1, stdout: r.stdout ?? "", stderr: r.stderr ?? "" };
|
||||
}
|
||||
|
||||
async function gatherContext(cost: { calls: number; tokens: number }): Promise<CycleContext> {
|
||||
// Last cycle: newest file in CYCLES_DIR.
|
||||
let lastCycleAt: string | null = null;
|
||||
let lastCycleGapId: string | null = null;
|
||||
try {
|
||||
const { readdir, stat } = await import("node:fs/promises");
|
||||
const entries = await readdir(CYCLES_DIR).catch(() => [] as string[]);
|
||||
let newestTs = 0;
|
||||
let newestPath: string | null = null;
|
||||
for (const e of entries) {
|
||||
if (!e.endsWith(".json")) continue;
|
||||
const p = join(CYCLES_DIR, e);
|
||||
const s = await stat(p);
|
||||
if (s.mtimeMs > newestTs) { newestTs = s.mtimeMs; newestPath = p; }
|
||||
}
|
||||
if (newestPath) {
|
||||
const r = JSON.parse(await readFile(newestPath, "utf8")) as CycleResult;
|
||||
lastCycleAt = r.started_at;
|
||||
lastCycleGapId = r.gap?.id ?? null;
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Autotune status — if the agent is busy, skip this cycle to avoid
|
||||
// contention on shared Ollama GPU memory.
|
||||
let autotuneBusy = false;
|
||||
try {
|
||||
const r = await fetch(`${GATEWAY_URL}/vectors/agent/status`, { signal: AbortSignal.timeout(2000) });
|
||||
if (r.ok) {
|
||||
const j = await r.json() as any;
|
||||
autotuneBusy = Boolean(j.running) && (j.queue_depth ?? 0) > 0;
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Dirty tree check.
|
||||
const status = git(["status", "--porcelain"]);
|
||||
const workingTreeDirty = status.code === 0 && status.stdout.trim().length > 0;
|
||||
|
||||
return {
|
||||
startedAt: new Date().toISOString(),
|
||||
dailyCallsUsed: cost.calls,
|
||||
dailyCallsBudget: DAILY_CALLS_BUDGET,
|
||||
dailyTokensUsed: cost.tokens,
|
||||
dailyTokensBudget: DAILY_TOKENS_BUDGET,
|
||||
lastCycleAt,
|
||||
lastCycleGapId,
|
||||
autotuneBusy,
|
||||
workingTreeDirty,
|
||||
};
|
||||
}
|
||||
|
||||
async function persistResult(res: CycleResult): Promise<void> {
|
||||
await mkdir(CYCLES_DIR, { recursive: true });
|
||||
await writeFile(join(CYCLES_DIR, `${res.cycle_id}.json`), JSON.stringify(res, null, 2));
|
||||
}
|
||||
|
||||
function sigHash(gap_id: string | null, proposalSummary: string): string {
|
||||
const h = createHash("sha256");
|
||||
h.update(gap_id ?? "no-gap");
|
||||
h.update("|");
|
||||
h.update(proposalSummary);
|
||||
return h.digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function emit(res: CycleResult): Promise<void> {
|
||||
return postEvent({
|
||||
source: "bot",
|
||||
cycle_id: res.cycle_id,
|
||||
sig_hash: sigHash(res.gap?.id ?? null, res.proposal?.summary ?? ""),
|
||||
event_kind: res.outcome,
|
||||
ok: res.outcome === "ok",
|
||||
staffer_id: res.proposal?.model_used,
|
||||
turns: res.cloud_calls,
|
||||
duration_ms: new Date(res.ended_at).getTime() - new Date(res.started_at).getTime(),
|
||||
extra: {
|
||||
reason: res.reason,
|
||||
pr_url: res.prUrl,
|
||||
files_added: res.filesAdded,
|
||||
files_updated: res.filesUpdated,
|
||||
files_noop: res.filesNoop,
|
||||
tests_green: res.testsGreen,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cycle_id = `bot-${new Date().toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
|
||||
const started_at = new Date().toISOString();
|
||||
log(`cycle ${cycle_id} starting`);
|
||||
|
||||
const finish = async (outcome: CycleOutcome, reason: string, extra: Partial<CycleResult> = {}): Promise<CycleResult> => {
|
||||
const res: CycleResult = {
|
||||
cycle_id,
|
||||
started_at,
|
||||
ended_at: new Date().toISOString(),
|
||||
outcome,
|
||||
reason,
|
||||
gap: null,
|
||||
proposal: null,
|
||||
filesAdded: [],
|
||||
filesUpdated: [],
|
||||
filesNoop: [],
|
||||
testsGreen: null,
|
||||
testsOutput: "",
|
||||
prUrl: null,
|
||||
tokens_used: 0,
|
||||
cloud_calls: 0,
|
||||
...extra,
|
||||
};
|
||||
await persistResult(res);
|
||||
await emit(res);
|
||||
log(`→ ${outcome}: ${reason}`);
|
||||
return res;
|
||||
};
|
||||
|
||||
// 1. Pause.
|
||||
if (await fileExists(PAUSE_FILE)) {
|
||||
return finish("skipped_pause", `pause file at ${PAUSE_FILE}`);
|
||||
}
|
||||
|
||||
// 2. Cost.
|
||||
const cost = await readCost();
|
||||
const bc = budgetCheck(cost);
|
||||
if (!bc.ok) return finish("skipped_cost", bc.reason);
|
||||
log(`cost: ${bc.reason}`);
|
||||
|
||||
// 3+4. Context + policy.shouldRunCycle.
|
||||
const ctx = await gatherContext(cost);
|
||||
const srq = policy.shouldRunCycle(ctx);
|
||||
if (!srq.run) {
|
||||
return finish(ctx.workingTreeDirty ? "skipped_dirty_tree" : "skipped_policy", srq.reason);
|
||||
}
|
||||
|
||||
// 5. Gaps + policy.pickGap.
|
||||
const gaps = await findGaps();
|
||||
log(`found ${gaps.length} [bot-eligible] gap(s) in PRD`);
|
||||
const gapPick = policy.pickGap(gaps);
|
||||
if (!gapPick.gap) return finish("skipped_no_gap", gapPick.reason);
|
||||
const gap = gapPick.gap;
|
||||
log(`picked gap ${gap.id}: "${gap.prd_line.slice(0, 80)}"`);
|
||||
|
||||
// 6. KB lookup + Proposal.
|
||||
// Bot cycles compound: prior cycles on this gap inform the current
|
||||
// proposal. Summary is a compact block injected into the cloud prompt
|
||||
// so the model can build on past successes and steer around past
|
||||
// failures. Empty string when this is the first cycle on this gap.
|
||||
const history = await loadHistory(gap.id, 5);
|
||||
const stats = statsFor(history);
|
||||
if (history.length > 0) {
|
||||
log(`kb: ${stats.attempts} prior cycle(s) — ${stats.pr_opened} pr / ${stats.tests_failed} tests_failed / ${stats.proposal_rejected} rejected / ${stats.noop} noop`);
|
||||
} else {
|
||||
log(`kb: first cycle on this gap`);
|
||||
}
|
||||
const historySummary = summarizeHistory(history);
|
||||
|
||||
let proposal: Proposal;
|
||||
try {
|
||||
proposal = await generateProposal(gap, historySummary);
|
||||
log(`proposal: ${proposal.summary} (${proposal.files.length} files, ~${proposal.estimated_loc} LOC, ${proposal.tokens_used} tokens)`);
|
||||
} catch (e) {
|
||||
await recordCost(1, 0);
|
||||
return finish("model_failed", (e as Error).message, { gap, cloud_calls: 1 });
|
||||
}
|
||||
await recordCost(1, proposal.tokens_used);
|
||||
|
||||
// 7. Score.
|
||||
const score = policy.scoreProposal(proposal);
|
||||
if (!score.accept) {
|
||||
return finish("proposal_rejected", score.reason, { gap, proposal, tokens_used: proposal.tokens_used, cloud_calls: 1 });
|
||||
}
|
||||
log(`proposal accepted: ${score.reason}`);
|
||||
|
||||
// 8. Branch.
|
||||
const dayStamp = new Date().toISOString().slice(0, 10).replace(/-/g, "");
|
||||
const branch = `bot/cycle-${dayStamp}-${gap.id}`;
|
||||
// Branch must be fresh. If one already exists from a prior failed cycle
|
||||
// for the same gap, bail — don't silently overwrite or reuse stale state.
|
||||
const check = git(["rev-parse", "--verify", `refs/heads/${branch}`]);
|
||||
if (check.code === 0) {
|
||||
return finish("apply_failed",
|
||||
`branch ${branch} already exists — delete it manually if you want a fresh cycle for this gap`,
|
||||
{ gap, proposal, tokens_used: proposal.tokens_used, cloud_calls: 1 });
|
||||
}
|
||||
const co = git(["checkout", "-b", branch]);
|
||||
if (co.code !== 0) {
|
||||
return finish("apply_failed", `git checkout -b failed: ${co.stderr}`,
|
||||
{ gap, proposal, tokens_used: proposal.tokens_used, cloud_calls: 1 });
|
||||
}
|
||||
log(`on branch ${branch}`);
|
||||
|
||||
// 9. Apply — Mem0 ADD/UPDATE/NOOP semantics.
|
||||
const ap = await applyProposal(proposal);
|
||||
if (ap.errors.length > 0) {
|
||||
git(["checkout", "main"]);
|
||||
git(["branch", "-D", branch]);
|
||||
return finish("apply_failed", ap.errors.join("; "),
|
||||
{ gap, proposal, tokens_used: proposal.tokens_used, cloud_calls: 1 });
|
||||
}
|
||||
const changed = [...ap.added, ...ap.updated];
|
||||
log(`apply: ${ap.added.length} add, ${ap.updated.length} update, ${ap.noop.length} noop`);
|
||||
|
||||
// Short-circuit: if every proposed file was identical to what's on
|
||||
// disk, there's nothing to test or PR. Clean up the branch and exit.
|
||||
if (changed.length === 0) {
|
||||
git(["checkout", "main"]);
|
||||
git(["branch", "-D", branch]);
|
||||
return finish("cycle_noop",
|
||||
`all ${proposal.files.length} proposed file(s) were identical to current state`,
|
||||
{ gap, proposal, filesNoop: ap.noop,
|
||||
tokens_used: proposal.tokens_used, cloud_calls: 1 });
|
||||
}
|
||||
|
||||
// 10. Test.
|
||||
log(`running tests (this will take a while)...`);
|
||||
const testRes = await runTests();
|
||||
if (!testRes.green) {
|
||||
git(["checkout", "."]);
|
||||
git(["checkout", "main"]);
|
||||
git(["branch", "-D", branch]);
|
||||
return finish("tests_failed", "cargo or bun test failed", {
|
||||
gap, proposal,
|
||||
filesAdded: ap.added, filesUpdated: ap.updated, filesNoop: ap.noop,
|
||||
testsGreen: false, testsOutput: testRes.output,
|
||||
tokens_used: proposal.tokens_used, cloud_calls: 1,
|
||||
});
|
||||
}
|
||||
log(`tests green`);
|
||||
|
||||
// 11. PR gate.
|
||||
const partial: CycleResult = {
|
||||
cycle_id, started_at,
|
||||
ended_at: new Date().toISOString(),
|
||||
outcome: "ok", reason: "",
|
||||
gap, proposal,
|
||||
filesAdded: ap.added, filesUpdated: ap.updated, filesNoop: ap.noop,
|
||||
testsGreen: true, testsOutput: testRes.output,
|
||||
prUrl: null, tokens_used: proposal.tokens_used, cloud_calls: 1,
|
||||
};
|
||||
const prGate = policy.shouldOpenPR(partial);
|
||||
if (!prGate.open) {
|
||||
git(["checkout", "main"]);
|
||||
git(["branch", "-D", branch]);
|
||||
return finish("pr_skipped_by_policy", prGate.reason, partial);
|
||||
}
|
||||
|
||||
// 12. Commit + push.
|
||||
git(["add", ...changed]);
|
||||
const commitMsg = [
|
||||
`bot: ${proposal.summary}`,
|
||||
"",
|
||||
proposal.rationale,
|
||||
"",
|
||||
`PRD gap: ${gap.prd_line}`,
|
||||
`Cycle: ${cycle_id}`,
|
||||
"",
|
||||
`Co-Authored-By: ${proposal.model_used} (cloud) <bot@lakehouse.local>`,
|
||||
].join("\n");
|
||||
const ci = git(["commit", "-m", commitMsg]);
|
||||
if (ci.code !== 0) {
|
||||
return finish("pr_failed", `git commit failed: ${ci.stderr}`, partial);
|
||||
}
|
||||
const ps = git(["push", "-u", "origin", branch]);
|
||||
if (ps.code !== 0) {
|
||||
return finish("pr_failed", `git push failed: ${ps.stderr}`, partial);
|
||||
}
|
||||
|
||||
// 13. Open PR.
|
||||
try {
|
||||
const pr = await openPr({
|
||||
branch,
|
||||
title: `bot: ${proposal.summary}`,
|
||||
body: [
|
||||
`**Gap** (PRD line ${gap.line_number}): ${gap.prd_line}`,
|
||||
"",
|
||||
`**Rationale**`,
|
||||
proposal.rationale,
|
||||
"",
|
||||
`**Cycle**: \`${cycle_id}\``,
|
||||
`**Model**: \`${proposal.model_used}\``,
|
||||
`**Tokens**: ${proposal.tokens_used}`,
|
||||
`**Added**: ${ap.added.map(f => `\`${f}\``).join(", ") || "_none_"}`,
|
||||
`**Updated**: ${ap.updated.map(f => `\`${f}\``).join(", ") || "_none_"}`,
|
||||
ap.noop.length > 0 ? `**NOOP (identical to current)**: ${ap.noop.map(f => `\`${f}\``).join(", ")}` : "",
|
||||
].join("\n"),
|
||||
});
|
||||
return finish("ok", `PR #${pr.number} opened`, { ...partial, prUrl: pr.html_url });
|
||||
} catch (e) {
|
||||
return finish("pr_failed", (e as Error).message, partial);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error("[bot] fatal:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
// Bot-local knowledge base. Every finished cycle already persists a
|
||||
// CycleResult to data/_bot/cycles/{id}.json — that IS the outcome log.
|
||||
// KB here just reads that dir, filters to prior cycles on the same gap,
|
||||
// and produces a short summary the cloud model can condition on.
|
||||
//
|
||||
// No separate jsonl, no new write path, no embedding calls. The bot's
|
||||
// "memory" is the same primary artifact that the observer consumes.
|
||||
//
|
||||
// Future: embedding-based neighbor matching across gaps (cheap once
|
||||
// sidecar is local), cross-pollination with scenario KB's
|
||||
// pathway_recommendations. Not required for the feedback loop to work
|
||||
// on a single gap — that's the floor we're building first.
|
||||
|
||||
import { readdir, readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { CycleResult } from "./types.ts";
|
||||
|
||||
const CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
|
||||
|
||||
export interface HistoryEntry {
|
||||
cycle_id: string;
|
||||
ended_at: string;
|
||||
outcome: string;
|
||||
reason: string;
|
||||
pr_url: string | null;
|
||||
tests_green: boolean | null;
|
||||
files_added: string[];
|
||||
files_updated: string[];
|
||||
tokens_used: number;
|
||||
}
|
||||
|
||||
export async function loadHistory(gap_id: string, max: number = 5): Promise<HistoryEntry[]> {
|
||||
let entries: string[] = [];
|
||||
try {
|
||||
entries = await readdir(CYCLES_DIR);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const matches: HistoryEntry[] = [];
|
||||
for (const e of entries) {
|
||||
if (!e.endsWith(".json")) continue;
|
||||
try {
|
||||
const raw = await readFile(join(CYCLES_DIR, e), "utf8");
|
||||
const r = JSON.parse(raw) as CycleResult;
|
||||
if (r.gap?.id !== gap_id) continue;
|
||||
matches.push({
|
||||
cycle_id: r.cycle_id,
|
||||
ended_at: r.ended_at,
|
||||
outcome: r.outcome,
|
||||
reason: r.reason,
|
||||
pr_url: r.prUrl,
|
||||
tests_green: r.testsGreen,
|
||||
files_added: r.filesAdded ?? [],
|
||||
files_updated: r.filesUpdated ?? [],
|
||||
tokens_used: r.tokens_used,
|
||||
});
|
||||
} catch {
|
||||
// Skip unreadable / malformed cycle files. Don't fail the current
|
||||
// cycle because an old one is corrupt.
|
||||
}
|
||||
}
|
||||
matches.sort((a, b) => b.ended_at.localeCompare(a.ended_at));
|
||||
return matches.slice(0, max);
|
||||
}
|
||||
|
||||
// Compact prompt-ready summary. Empty string when there's no history —
|
||||
// caller can skip the "prior attempts" block entirely.
|
||||
export function summarizeHistory(h: HistoryEntry[]): string {
|
||||
if (h.length === 0) return "";
|
||||
const lines = h.map(e => {
|
||||
const when = e.ended_at.slice(0, 16).replace("T", " ");
|
||||
const files = [...e.files_added, ...e.files_updated];
|
||||
const filesStr = files.length > 0 ? ` touched: ${files.join(", ")}` : "";
|
||||
const prStr = e.pr_url ? ` PR: ${e.pr_url}` : "";
|
||||
return `- ${when} UTC — ${e.outcome}${prStr}${filesStr}\n reason: ${e.reason}`;
|
||||
});
|
||||
return [
|
||||
`Prior attempts on this gap (${h.length} most recent):`,
|
||||
...lines,
|
||||
"",
|
||||
"Learn from these: build on what worked, avoid paths that failed.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
// Aggregate stats for telemetry — lets the bot expose "% of cycles on
|
||||
// this gap that landed a PR" without re-parsing the raw history.
|
||||
export function statsFor(h: HistoryEntry[]): {
|
||||
attempts: number;
|
||||
pr_opened: number;
|
||||
tests_failed: number;
|
||||
proposal_rejected: number;
|
||||
noop: number;
|
||||
} {
|
||||
return {
|
||||
attempts: h.length,
|
||||
pr_opened: h.filter(e => e.outcome === "ok").length,
|
||||
tests_failed: h.filter(e => e.outcome === "tests_failed").length,
|
||||
proposal_rejected: h.filter(e => e.outcome === "proposal_rejected").length,
|
||||
noop: h.filter(e => e.outcome === "cycle_noop").length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// POST bot cycle outcomes to the observer on :3800 so they accumulate
|
||||
// in the KB alongside scenario + autotune events. Non-fatal on failure:
|
||||
// the cycle result is also written to disk so observability isn't a
|
||||
// single point of failure.
|
||||
|
||||
const OBSERVER_URL = process.env.LH_OBSERVER_URL ?? "http://localhost:3800";
|
||||
|
||||
export interface BotObserverEvent {
|
||||
source: "bot";
|
||||
cycle_id: string;
|
||||
sig_hash: string; // stable hash of the gap + proposal for dedup
|
||||
event_kind: string; // cycle outcome (ok, tests_failed, ...)
|
||||
ok: boolean;
|
||||
staffer_id?: string; // the model used, e.g. "gpt-oss:120b"
|
||||
turns?: number; // number of cloud calls this cycle
|
||||
duration_ms?: number;
|
||||
extra?: Record<string, any>;
|
||||
}
|
||||
|
||||
export async function postEvent(ev: BotObserverEvent): Promise<void> {
|
||||
try {
|
||||
const r = await fetch(`${OBSERVER_URL}/event`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(ev),
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
if (!r.ok) {
|
||||
console.error(`[bot/observer] ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[bot/observer] POST failed: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// YOU WRITE THIS FILE. These four functions shape the bot's entire
|
||||
// behavior. Defaults below are safe and intentionally conservative —
|
||||
// redline to taste. Each function is ~5-10 lines.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
import type { CycleContext, CycleResult, Gap, Proposal } from "./types.ts";
|
||||
|
||||
// Should this cycle even fire right now?
|
||||
// Called AFTER the pause-file check + cost-cap check have already passed.
|
||||
// Use this for judgment calls (cooldown, shared-resource conflicts).
|
||||
export function shouldRunCycle(ctx: CycleContext): { run: boolean; reason: string } {
|
||||
if (ctx.workingTreeDirty) {
|
||||
return { run: false, reason: "dirty working tree — commit or stash your work first" };
|
||||
}
|
||||
if (ctx.lastCycleAt) {
|
||||
const minsAgo = (Date.now() - new Date(ctx.lastCycleAt).getTime()) / 60000;
|
||||
if (minsAgo < 10) return { run: false, reason: `last cycle ${minsAgo.toFixed(1)}min ago (<10min cooldown)` };
|
||||
}
|
||||
if (ctx.autotuneBusy) {
|
||||
return { run: false, reason: "autotune agent is running — avoid Ollama contention" };
|
||||
}
|
||||
return { run: true, reason: "ready" };
|
||||
}
|
||||
|
||||
// Which [bot-eligible] gap from the PRD does this cycle attack?
|
||||
// Default: first in document order. Alternatives: random for exploration,
|
||||
// smallest-context, tag-priority (e.g. [bot-eligible:high] wins).
|
||||
export function pickGap(gaps: Gap[]): { gap: Gap | null; reason: string } {
|
||||
if (gaps.length === 0) return { gap: null, reason: "no [bot-eligible] gaps in PRD" };
|
||||
return { gap: gaps[0], reason: "first eligible gap in document order" };
|
||||
}
|
||||
|
||||
// Cheap gate BEFORE burning test cycles. Reject if proposal smells wrong.
|
||||
// This catches runaway cloud proposals before we apply+test them.
|
||||
export function scoreProposal(p: Proposal): { accept: boolean; reason: string } {
|
||||
if (p.files.length === 0) return { accept: false, reason: "empty proposal" };
|
||||
if (p.files.length > 5) return { accept: false, reason: `>5 files (${p.files.length})` };
|
||||
if (p.estimated_loc > 200) return { accept: false, reason: `>200 LOC (${p.estimated_loc})` };
|
||||
|
||||
// Paths the bot is NEVER allowed to touch. Extend cautiously.
|
||||
const forbidden = [
|
||||
".git/", "secrets", "lakehouse.toml",
|
||||
"docs/ADR-", "docs/DECISIONS.md", "docs/PRD.md",
|
||||
"/etc/", "/root/", "Cargo.lock",
|
||||
];
|
||||
for (const f of p.files) {
|
||||
const hit = forbidden.find(prefix => f.path.includes(prefix));
|
||||
if (hit) return { accept: false, reason: `touches forbidden path (${hit}): ${f.path}` };
|
||||
}
|
||||
return { accept: true, reason: "passes size + path filters" };
|
||||
}
|
||||
|
||||
// Tests are green and changes applied. Do we actually open a PR?
|
||||
// Mem0-aligned: the cycle already short-circuited with `cycle_noop` if
|
||||
// every file was identical to current state, so by the time we're here
|
||||
// at least one file was actually added or updated.
|
||||
//
|
||||
// Stricter options below — uncomment to require a test file touched, etc.
|
||||
export function shouldOpenPR(c: CycleResult): { open: boolean; reason: string } {
|
||||
if (!c.testsGreen) return { open: false, reason: "tests red" };
|
||||
const realChanges = c.filesAdded.length + c.filesUpdated.length;
|
||||
if (realChanges === 0) return { open: false, reason: "nothing actually changed on disk" };
|
||||
|
||||
// Stricter option — require at least one test file in the diff:
|
||||
// const hasTest = [...c.filesAdded, ...c.filesUpdated].some(f => /test|_tests/i.test(f));
|
||||
// if (!hasTest) return { open: false, reason: "no test file touched" };
|
||||
|
||||
return { open: true, reason: `green + ${c.filesAdded.length} add / ${c.filesUpdated.length} update` };
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Gitea PR open. Reads PAT from ~/.git-credentials (set up by the
|
||||
// credential-helper flow). All PRs open as DRAFT so J reviews before
|
||||
// merge — never auto-merge.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
const GITEA_HOST = "https://git.agentview.dev";
|
||||
const REPO_OWNER = "profit";
|
||||
const REPO_NAME = "lakehouse";
|
||||
const CRED_FILE = "/home/profit/.git-credentials";
|
||||
|
||||
async function readPat(): Promise<string> {
|
||||
const raw = await readFile(CRED_FILE, "utf8");
|
||||
for (const line of raw.split("\n")) {
|
||||
const m = line.match(/^https:\/\/[^:]+:([^@]+)@git\.agentview\.dev/);
|
||||
if (m) return m[1];
|
||||
}
|
||||
throw new Error(`no Gitea PAT found in ${CRED_FILE}`);
|
||||
}
|
||||
|
||||
export interface OpenPrInput {
|
||||
branch: string; // head branch (just pushed)
|
||||
base?: string; // default "main"
|
||||
title: string;
|
||||
body: string;
|
||||
}
|
||||
|
||||
export interface OpenPrResult {
|
||||
number: number;
|
||||
html_url: string;
|
||||
}
|
||||
|
||||
export async function openPr(input: OpenPrInput): Promise<OpenPrResult> {
|
||||
const pat = await readPat();
|
||||
const url = `${GITEA_HOST}/api/v1/repos/${REPO_OWNER}/${REPO_NAME}/pulls`;
|
||||
const payload = {
|
||||
title: input.title,
|
||||
body: input.body,
|
||||
head: input.branch,
|
||||
base: input.base ?? "main",
|
||||
// Gitea uses "draft" flag — always on for bot PRs.
|
||||
draft: true,
|
||||
};
|
||||
const r = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"Authorization": `token ${pat}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!r.ok) {
|
||||
throw new Error(`Gitea ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
const j = await r.json() as any;
|
||||
return { number: j.number, html_url: j.html_url };
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
// Gap detection + cloud proposal.
|
||||
//
|
||||
// Gap detection: scan docs/PRD.md for lines tagged [bot-eligible].
|
||||
// Each match becomes a Gap with surrounding context.
|
||||
//
|
||||
// Proposal: one-shot call to the T3 cloud model via the Python sidecar's
|
||||
// /generate endpoint. Asks for a structured JSON response with file
|
||||
// contents. Truncation-resistant via Phase 21's generate_continuable —
|
||||
// for now we pass max_tokens high and rely on the model completing in
|
||||
// one pass; swap to the Rust continuation wrapper if we see truncation.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Gap, Proposal } from "./types.ts";
|
||||
|
||||
const SIDECAR_URL = process.env.LH_SIDECAR_URL ?? "http://localhost:3200";
|
||||
const REPO_ROOT = "/home/profit/lakehouse";
|
||||
const PRD_PATH = `${REPO_ROOT}/docs/PRD.md`;
|
||||
const CLOUD_MODEL = process.env.LH_BOT_MODEL ?? "gpt-oss:120b";
|
||||
const MAX_TOKENS = 6000;
|
||||
|
||||
export async function findGaps(): Promise<Gap[]> {
|
||||
const prd = await readFile(PRD_PATH, "utf8");
|
||||
const lines = prd.split("\n");
|
||||
const gaps: Gap[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (!lines[i].includes("[bot-eligible]")) continue;
|
||||
const contextLines = lines.slice(i, Math.min(i + 6, lines.length)).join("\n");
|
||||
const id = createHash("sha256").update(lines[i]).digest("hex").slice(0, 12);
|
||||
gaps.push({
|
||||
id,
|
||||
prd_line: lines[i].trim(),
|
||||
context: contextLines,
|
||||
source_file: "docs/PRD.md",
|
||||
line_number: i + 1,
|
||||
});
|
||||
}
|
||||
return gaps;
|
||||
}
|
||||
|
||||
const SYSTEM_PROMPT = `You are an assistant that proposes small, testable code changes to the Lakehouse repo.
|
||||
The Lakehouse is a Rust-first data platform with 13 crates + Bun/TypeScript test harness.
|
||||
You will be given one PRD gap tagged [bot-eligible] and must respond with a STRICT JSON object — no prose.
|
||||
|
||||
Rules:
|
||||
- Response MUST be a single JSON object, no markdown fences, no commentary.
|
||||
- Change MUST be small: <200 lines total, ≤5 files.
|
||||
- Include at least one test file (new or modified) that proves the change.
|
||||
- NEVER touch .git/, secrets, lakehouse.toml, docs/ADR-*, docs/DECISIONS.md, docs/PRD.md, /etc/, /root/, Cargo.lock.
|
||||
- Paths MUST be repo-relative (no leading /).
|
||||
- Whole-file contents only — no patches, no diffs.
|
||||
|
||||
Response shape:
|
||||
{
|
||||
"summary": "one line",
|
||||
"rationale": "why this addresses the gap",
|
||||
"files": [ { "path": "crates/foo/src/bar.rs", "content": "<full file>", "is_new": false } ],
|
||||
"estimated_loc": 42
|
||||
}`;
|
||||
|
||||
export async function generateProposal(gap: Gap, historySummary: string = ""): Promise<Proposal> {
|
||||
const sections = [
|
||||
`PRD gap (line ${gap.line_number}):`,
|
||||
"```",
|
||||
gap.context,
|
||||
"```",
|
||||
"",
|
||||
];
|
||||
if (historySummary) {
|
||||
sections.push(historySummary, "");
|
||||
}
|
||||
sections.push("Propose a small change that addresses this gap. Respond with the JSON object only.");
|
||||
const userPrompt = sections.join("\n");
|
||||
|
||||
const r = await fetch(`${SIDECAR_URL}/generate`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: CLOUD_MODEL,
|
||||
system: SYSTEM_PROMPT,
|
||||
prompt: userPrompt,
|
||||
temperature: 0.2,
|
||||
max_tokens: MAX_TOKENS,
|
||||
think: false,
|
||||
}),
|
||||
signal: AbortSignal.timeout(180000), // cloud T3 can be slow — 3 min
|
||||
});
|
||||
if (!r.ok) {
|
||||
throw new Error(`sidecar ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
const j = await r.json() as any;
|
||||
const raw: string = j.text ?? j.response ?? "";
|
||||
const usage = j.usage ?? {};
|
||||
const tokens = (usage.prompt_tokens ?? 0) + (usage.completion_tokens ?? 0);
|
||||
|
||||
const parsed = extractJson(raw);
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
throw new Error(`model returned no JSON object. Raw head: ${raw.slice(0, 300)}`);
|
||||
}
|
||||
if (!Array.isArray(parsed.files)) {
|
||||
throw new Error(`proposal.files not an array: ${JSON.stringify(parsed).slice(0, 200)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
summary: String(parsed.summary ?? "").trim(),
|
||||
rationale: String(parsed.rationale ?? "").trim(),
|
||||
files: parsed.files.map((f: any) => ({
|
||||
path: String(f.path ?? ""),
|
||||
content: String(f.content ?? ""),
|
||||
is_new: Boolean(f.is_new),
|
||||
})).filter((f: any) => f.path && f.content !== undefined),
|
||||
estimated_loc: Number(parsed.estimated_loc ?? 0),
|
||||
model_used: CLOUD_MODEL,
|
||||
tokens_used: tokens,
|
||||
};
|
||||
}
|
||||
|
||||
// Find the first balanced JSON object in the string. Tolerates leading
|
||||
// "```json" fences even though we asked the model not to emit them.
|
||||
function extractJson(text: string): any | null {
|
||||
const cleaned = text.replace(/^```(?:json)?\s*/i, "").replace(/```\s*$/i, "").trim();
|
||||
let depth = 0;
|
||||
let start = -1;
|
||||
for (let i = 0; i < cleaned.length; i++) {
|
||||
const c = cleaned[i];
|
||||
if (c === "{") {
|
||||
if (depth === 0) start = i;
|
||||
depth++;
|
||||
} else if (c === "}") {
|
||||
depth--;
|
||||
if (depth === 0 && start >= 0) {
|
||||
try {
|
||||
return JSON.parse(cleaned.slice(start, i + 1));
|
||||
} catch {
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
// Run the test gate: `cargo test --workspace` + `bun test`.
|
||||
// Returns { green, output } — output is last ~4KB of combined stdout/stderr.
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const REPO_ROOT = "/home/profit/lakehouse";
|
||||
const TEST_TIMEOUT_MS = 15 * 60 * 1000; // 15 min cargo + 2 min bun, generous
|
||||
|
||||
export async function runTests(): Promise<{ green: boolean; output: string }> {
|
||||
const cargoOut = await runCmd("cargo", ["test", "--workspace", "--quiet", "--", "--test-threads=1"], REPO_ROOT, TEST_TIMEOUT_MS);
|
||||
if (cargoOut.code !== 0) {
|
||||
return { green: false, output: tail(`cargo test (${cargoOut.code}):\n${cargoOut.combined}`) };
|
||||
}
|
||||
|
||||
const bunOut = await runCmd("bun", ["test", "tests/multi-agent"], REPO_ROOT, 120000);
|
||||
const bunGreen = bunOut.code === 0;
|
||||
const combined = [
|
||||
`cargo test: OK`,
|
||||
`bun test (${bunOut.code}):`,
|
||||
bunOut.combined,
|
||||
].join("\n");
|
||||
return { green: bunGreen, output: tail(combined) };
|
||||
}
|
||||
|
||||
function tail(s: string, n = 4096): string {
|
||||
return s.length > n ? "…" + s.slice(-n) : s;
|
||||
}
|
||||
|
||||
function runCmd(
|
||||
cmd: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
timeoutMs: number,
|
||||
): Promise<{ code: number; combined: string }> {
|
||||
return new Promise(resolveP => {
|
||||
const child = spawn(cmd, args, { cwd, env: { ...process.env } });
|
||||
let combined = "";
|
||||
child.stdout.on("data", d => { combined += d.toString(); });
|
||||
child.stderr.on("data", d => { combined += d.toString(); });
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
||||
child.on("close", code => {
|
||||
clearTimeout(timer);
|
||||
resolveP({ code: code ?? -1, combined });
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
// Shared types for the PR-bot. Small and explicit — the bot's behavior
|
||||
// should be readable from these shapes alone.
|
||||
|
||||
export interface Gap {
|
||||
id: string; // hash of prd_line — stable key for dedup
|
||||
prd_line: string; // the [bot-eligible]-tagged line, verbatim
|
||||
context: string; // surrounding context from the PRD (next ~5 lines)
|
||||
source_file: string; // e.g. "docs/PRD.md"
|
||||
line_number: number;
|
||||
}
|
||||
|
||||
export interface ProposalFile {
|
||||
path: string; // repo-relative (no leading slash)
|
||||
content: string; // full file content — the bot writes whole files, not patches
|
||||
is_new: boolean; // true = create, false = overwrite existing
|
||||
}
|
||||
|
||||
export interface Proposal {
|
||||
summary: string; // one-line
|
||||
rationale: string; // why this change addresses the gap
|
||||
files: ProposalFile[];
|
||||
estimated_loc: number; // total added+changed lines across all files
|
||||
model_used: string;
|
||||
tokens_used: number;
|
||||
}
|
||||
|
||||
export interface CycleContext {
|
||||
startedAt: string; // ISO
|
||||
dailyCallsUsed: number;
|
||||
dailyCallsBudget: number;
|
||||
dailyTokensUsed: number;
|
||||
dailyTokensBudget: number;
|
||||
lastCycleAt: string | null;
|
||||
lastCycleGapId: string | null;
|
||||
autotuneBusy: boolean;
|
||||
workingTreeDirty: boolean;
|
||||
}
|
||||
|
||||
export type CycleOutcome =
|
||||
| "ok" // PR opened
|
||||
| "skipped_pause"
|
||||
| "skipped_cost"
|
||||
| "skipped_policy"
|
||||
| "skipped_no_gap"
|
||||
| "skipped_dirty_tree"
|
||||
| "proposal_rejected"
|
||||
| "apply_failed"
|
||||
| "cycle_noop" // proposal applied but every file was identical to what's on disk
|
||||
| "tests_failed"
|
||||
| "pr_skipped_by_policy"
|
||||
| "pr_failed"
|
||||
| "model_failed";
|
||||
|
||||
// Mem0-aligned apply outcomes. Three shapes instead of the binary
|
||||
// "written" / "errored". NOOP is a first-class outcome — identical
|
||||
// content shouldn't waste test cycles or open an empty PR.
|
||||
export type ApplyMode = "add" | "update" | "noop";
|
||||
|
||||
export interface ApplyOutcome {
|
||||
added: string[];
|
||||
updated: string[];
|
||||
noop: string[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
export interface CycleResult {
|
||||
cycle_id: string;
|
||||
started_at: string;
|
||||
ended_at: string;
|
||||
outcome: CycleOutcome;
|
||||
reason: string;
|
||||
gap: Gap | null;
|
||||
proposal: Proposal | null;
|
||||
filesAdded: string[];
|
||||
filesUpdated: string[];
|
||||
filesNoop: string[];
|
||||
testsGreen: boolean | null; // null = not run
|
||||
testsOutput: string;
|
||||
prUrl: string | null;
|
||||
tokens_used: number;
|
||||
cloud_calls: number;
|
||||
}
|
||||
|
||||
export interface CostState {
|
||||
date: string; // YYYY-MM-DD UTC
|
||||
calls: number;
|
||||
tokens: number;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"dependencies": {
|
||||
"langfuse": "^3.38.20",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"langfuse": ["langfuse@3.38.20", "", { "dependencies": { "langfuse-core": "^3.38.20" } }, "sha512-MAmBAASSzJtmK1O9HQegA1mFsQhT8Yf+OJRGvE7FXkyv3g/eiBE0glLD0Ohg3pkxhoPdggM5SejK7ue9ctlaMA=="],
|
||||
|
||||
"langfuse-core": ["langfuse-core@3.38.20", "", { "dependencies": { "mustache": "^4.2.0" } }, "sha512-zBKVmQN/1oT5VWZUBYlWzvokIlkC/6mnpgr/2atMyTeAm+jR3ia7w2iJMjlrF5/oG8ukO1s8+LDRCzJpF1QeEA=="],
|
||||
|
||||
"mustache": ["mustache@4.2.0", "", { "bin": { "mustache": "bin/mustache" } }, "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ=="],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
{
|
||||
"_description": "Lakehouse model matrix — authoritative routing for all agent tiers. Local models do the heavy lifting; cloud models are consulted sparingly for overview, strategic, and gatekeeper decisions. Read by tests/multi-agent/scenario.ts and mcp-server/index.ts.",
|
||||
"version": 1,
|
||||
"updated": "2026-04-21",
|
||||
"providers": {
|
||||
"ollama_local": {
|
||||
"base_url": "http://localhost:11434",
|
||||
"key_env": null
|
||||
},
|
||||
"ollama_cloud": {
|
||||
"base_url": "https://ollama.com",
|
||||
"key_env": "OLLAMA_CLOUD_KEY",
|
||||
"key_source": "/root/llm_team_config.json → providers.ollama_cloud.api_key",
|
||||
"rate_budget": {
|
||||
"calls_per_hour": 200,
|
||||
"calls_per_day": 2000,
|
||||
"notes": "Paid tier — generous. Policy: keep overview calls ≤ 3/scenario, strategic ≤ 10/day, gatekeeper ≤ 5/day."
|
||||
}
|
||||
}
|
||||
},
|
||||
"tiers": {
|
||||
"t1_hot": {
|
||||
"purpose": "Per tool call — SQL generation, hybrid_search, sql(). Runs 50-200 times per scenario. Latency-sensitive.",
|
||||
"kind": "local_fast",
|
||||
"primary": { "model": "qwen3.5:latest", "provider": "ollama_local", "context_window": 262144 },
|
||||
"fallback": { "model": "qwen2.5:latest", "provider": "ollama_local", "context_window": 32768 },
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.3,
|
||||
"never_route_cloud": true,
|
||||
"context_budget": {
|
||||
"system_prompt_cap": 4000,
|
||||
"prior_context_cap": 6000,
|
||||
"tool_results_cap": 8000,
|
||||
"safety_margin": 2000,
|
||||
"overflow_policy": "summarize_oldest_tool_results_via_t3"
|
||||
},
|
||||
"rationale": "qwen3.5:latest is a 9.7B thinking model with 262K context that emits clean JSON (verified 2026-04-21 after mistral A/B wipeout). Thinking budget requires max_tokens ≥ 400; we hold 1000 to keep propose_done payloads intact. qwen2.5 stays as fallback."
|
||||
},
|
||||
"t2_review": {
|
||||
"purpose": "Per step consensus — executor ↔ reviewer loop critique. 5-14 calls per event.",
|
||||
"kind": "local_balanced",
|
||||
"primary": { "model": "qwen3:latest", "provider": "ollama_local", "context_window": 40960 },
|
||||
"fallback": { "model": "qwen2.5:latest", "provider": "ollama_local", "context_window": 32768 },
|
||||
"max_tokens": 800,
|
||||
"temperature": 0.3,
|
||||
"never_route_cloud": true,
|
||||
"context_budget": {
|
||||
"system_prompt_cap": 2000,
|
||||
"recent_turns_cap": 4000,
|
||||
"safety_margin": 1000
|
||||
},
|
||||
"rationale": "Reviewer only needs to detect schema violations and drift — a 7B model is sufficient."
|
||||
},
|
||||
"t3_overview": {
|
||||
"purpose": "Mid-day checkpoint after every misplacement + every Nth event, and cross-day lesson. 1-3 calls per scenario.",
|
||||
"kind": "thinking_cloud",
|
||||
"primary": { "model": "gpt-oss:120b", "provider": "ollama_cloud", "context_window": 131072 },
|
||||
"local_fallback": { "model": "gpt-oss:20b", "provider": "ollama_local", "context_window": 131072 },
|
||||
"max_tokens": 900,
|
||||
"temperature": 0.2,
|
||||
"cloud_budget_per_scenario": 5,
|
||||
"env_flag": "LH_OVERVIEW_CLOUD=1",
|
||||
"context_budget": {
|
||||
"event_digest_cap": 30000,
|
||||
"checkpoint_cap": 8000,
|
||||
"lesson_corpus_cap": 40000,
|
||||
"safety_margin": 8000,
|
||||
"overflow_policy": "chunk_lessons_via_cosine_topk"
|
||||
},
|
||||
"rationale": "Same prompt family as local 20b (gpt-oss series) — prompts port directly. 120b is faster via cloud than 20b local in practice, and lessons are noticeably more specific."
|
||||
},
|
||||
"t4_strategic": {
|
||||
"purpose": "Daily playbook board re-ranking, weekly gap audit, pattern discovery across accumulated playbooks. 1-10 calls per day.",
|
||||
"kind": "thinking_cloud_large",
|
||||
"primary": { "model": "kimi-k2.6", "provider": "ollama_cloud", "context_window": 200000 },
|
||||
"secondary": { "model": "qwen3.5:397b", "provider": "ollama_cloud", "context_window": 131072 },
|
||||
"fallback": { "model": "glm-4.7", "provider": "ollama_cloud", "context_window": 131072 },
|
||||
"local_fallback": { "model": "gpt-oss:20b", "provider": "ollama_local", "context_window": 131072 },
|
||||
"max_tokens": 2000,
|
||||
"temperature": 0.2,
|
||||
"cloud_budget_per_day": 10,
|
||||
"context_budget": {
|
||||
"playbook_corpus_cap": 80000,
|
||||
"pattern_history_cap": 20000,
|
||||
"safety_margin": 16000,
|
||||
"overflow_policy": "two_pass_map_reduce"
|
||||
},
|
||||
"rationale": "J named qwen3.5 specifically. GLM-4.7 is a promising alternate for debate phase. Runs after all scenarios complete for the day."
|
||||
},
|
||||
"t5_gatekeeper": {
|
||||
"purpose": "MUST route here: architecture changes, new client onboarding, schema migrations, playbook retirements, index rebuilds, autotune config changes.",
|
||||
"kind": "thinking_cloud_deepest",
|
||||
"primary": { "model": "kimi-k2-thinking", "provider": "ollama_cloud", "context_window": 200000 },
|
||||
"fallback": { "model": "deepseek-v3.1:671b", "provider": "ollama_cloud", "context_window": 131072 },
|
||||
"secondary_fallback": { "model": "qwen3.5:397b", "provider": "ollama_cloud", "context_window": 131072 },
|
||||
"local_fallback": { "model": "gpt-oss:20b", "provider": "ollama_local", "context_window": 131072 },
|
||||
"max_tokens": 4000,
|
||||
"temperature": 0.1,
|
||||
"cloud_budget_per_day": 5,
|
||||
"audit_log": true,
|
||||
"context_budget": {
|
||||
"decision_doc_cap": 50000,
|
||||
"evidence_bundle_cap": 100000,
|
||||
"prior_gatekeeper_decisions_cap": 20000,
|
||||
"safety_margin": 20000,
|
||||
"overflow_policy": "escalate_to_kimi_k2_1t_or_split_decision"
|
||||
},
|
||||
"rationale": "Highest-stakes decisions — reasoning depth matters more than latency. Audit log so J can always see what the gatekeeper was asked and what it answered. No human approval required today; escalate later if mis-decisions show up."
|
||||
}
|
||||
},
|
||||
|
||||
"embeddings": {
|
||||
"_description": "Vector embedding models for hybrid search + playbook_memory similarity. Always local — embedding calls are too high-volume for cloud. qwen3-embedding is the new primary (2026-04-21); nomic-embed-text-v2-moe is the MoE alternative for corpus-scale embedding; original nomic-embed-text stays as legacy fallback while vectord indexes are being re-embedded.",
|
||||
"primary": { "model": "qwen3-embedding", "provider": "ollama_local", "dim": 1024, "context_window": 32768 },
|
||||
"alternate": { "model": "nomic-embed-text-v2-moe", "provider": "ollama_local", "dim": 768, "context_window": 2048, "notes": "MoE variant — better for long documents, slower per-call, roughly matches nomic-embed-text on short text." },
|
||||
"legacy": { "model": "nomic-embed-text:latest", "provider": "ollama_local", "dim": 768, "context_window": 8192, "notes": "Used by existing vectord indexes. Do NOT switch live until indexes are rebuilt, or hybrid search recall will crater." },
|
||||
"migration_plan": "Re-embed workers_500k (and all production indexes) with qwen3-embedding, keep the old index under a _v1 suffix for rollback, flip the profile pointer only after recall benchmarks match or exceed legacy. Touches vectord::agent + autotune."
|
||||
},
|
||||
|
||||
"context_management": {
|
||||
"_description": "Rule zero: NEVER call a model with more tokens than its context_window minus safety_margin. Every call goes through the budget checker first. If over budget → chunk, summarize, or escalate. This is the stability floor.",
|
||||
"token_estimator": {
|
||||
"method": "chars_div_4",
|
||||
"note": "Rough, biased safe by ~15%. For production, swap to tiktoken or the provider's tokenizer endpoint."
|
||||
},
|
||||
"overflow_policies": {
|
||||
"summarize_oldest_tool_results_via_t3": {
|
||||
"when": "T1 conversation history + tool results exceed context_budget.tool_results_cap",
|
||||
"how": "Send oldest N tool results to T3 with prompt 'summarize these into 500 tokens that preserve what the executor needs to know'; replace them with the summary in the running conversation."
|
||||
},
|
||||
"chunk_lessons_via_cosine_topk": {
|
||||
"when": "lesson corpus in data/_playbook_lessons/*.json exceeds lesson_corpus_cap",
|
||||
"how": "Embed the current scenario spec, cosine-rank all lessons, take top-K until budget exhausted. Fall back to date-sorted if embeddings unavailable."
|
||||
},
|
||||
"two_pass_map_reduce": {
|
||||
"when": "T4 playbook corpus exceeds playbook_corpus_cap",
|
||||
"how": "Pass 1: chunk playbooks into ≤30K token shards, run primary model on each shard to emit 'shard summary'. Pass 2: feed all summaries to primary model for global synthesis. Logged as two audit entries."
|
||||
},
|
||||
"escalate_to_kimi_k2_1t_or_split_decision": {
|
||||
"when": "T5 decision evidence exceeds decision_doc_cap + evidence_bundle_cap",
|
||||
"how": "Prefer kimi-k2:1t which has 1M context. If still over, split decision into sub-decisions (e.g. 'retire playbooks by city' instead of 'retire playbooks globally') and loop."
|
||||
}
|
||||
},
|
||||
"chunking_cache": {
|
||||
"_description": "Precomputed shards of the playbook corpus, indexed by (corpus_version, shard_id). Avoids re-chunking on every T4 run.",
|
||||
"location": "data/_chunk_cache/",
|
||||
"invalidation": "Key includes corpus hash. When playbook_memory changes, the hash changes, the cache misses, and chunks regenerate.",
|
||||
"implementation_status": "SPEC — next sprint."
|
||||
},
|
||||
"implementation_status": "context_window + context_budget fields WIRED in config. Enforcement helper NOT yet wired in agent.ts. Next: add estimateTokens() + budgetCheck() helpers, route all generate() calls through them, emit a metric when overflow policy fires."
|
||||
},
|
||||
"experimental_rotation": {
|
||||
"enabled": false,
|
||||
"purpose": "Sample newer models on a schedule to collect comparison data without rate-limit risk.",
|
||||
"candidates": [
|
||||
{ "model": "minimax-m2.7", "notes": "Newer minimax; unknown output stability" },
|
||||
{ "model": "glm-5", "notes": "GLM next-gen; larger context" },
|
||||
{ "model": "glm-5.1", "notes": "Incremental on GLM-5" },
|
||||
{ "model": "qwen3-next:80b", "notes": "Qwen's experimental successor; smaller than 3.5" },
|
||||
{ "model": "qwen3-coder-next", "notes": "Coder-optimized — good for SQL gen T1 experiments" },
|
||||
{ "model": "deepseek-v3.2", "notes": "Smaller deepseek; reasoning/coding" },
|
||||
{ "model": "nemotron-3-super", "notes": "NVIDIA 230B; general" },
|
||||
{ "model": "cogito-2.1:671b", "notes": "671B general" },
|
||||
{ "model": "mistral-large-3:675b", "notes": "Mistral's flagship; good T3 candidate" }
|
||||
],
|
||||
"rotation": "weekly",
|
||||
"sample_rate": 0.1,
|
||||
"apply_to_tier": "t4_strategic",
|
||||
"notes": "When enabled, T4 routes 10% of calls to a rotating experimental model. Log comparison in /data/_model_eval/ — if the experimental consistently beats primary across 3 rotations, promote it to primary."
|
||||
},
|
||||
"playbook_versioning": {
|
||||
"enabled": true,
|
||||
"purpose": "A playbook can work, then break when architecture changes. Versioning lets us pin which change retired which playbook.",
|
||||
"dataset": "playbook_memory",
|
||||
"schema_additions": {
|
||||
"version": "integer — auto-increment per operation",
|
||||
"parent_id": "string — previous version entry_id for same operation (null for v1)",
|
||||
"retired_at": "timestamp — set when success_rate drops or architecture changes",
|
||||
"retirement_reason": "string — e.g. 'schema_migration:workers_500k 2026-05-03'",
|
||||
"architecture_snapshot": "object — crate versions, index name, schema fingerprint at seed time"
|
||||
},
|
||||
"retire_triggers": [
|
||||
"success_rate < 0.3 over last 20 citations",
|
||||
"schema_fingerprint mismatch detected at retrieval time",
|
||||
"architecture change event emitted by ingestd/vectord",
|
||||
"T5 gatekeeper explicitly retires via /vectors/playbook_memory/retire"
|
||||
],
|
||||
"read_back_policy": "retrieval returns only non-retired versions. History endpoint /vectors/playbook_memory/history/{operation} returns the full chain.",
|
||||
"ui_surface": "mcp-server to render a diff view: side-by-side of playbook versions with a timeline of what changed and when.",
|
||||
"implementation_status": "SPEC — not yet wired. Target: next sprint. Touches gateway + catalogd + mcp-server."
|
||||
},
|
||||
"matrix_index_hybrid_search_note": "Phase 22 candidate: elevate the hybrid_search T1 tool to consult T3 when a pool returns <3 matches OR when the same (role, city) combo has failed N times in 24h. Consult result is a reformulated sql_filter the executor retries with. Keeps T1 fast on the happy path, escalates to T3 only on low-recall signals."
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
# Phase 39: Provider Registry
|
||||
#
|
||||
# Per-provider base_url, auth scheme, and default model. The gateway's
|
||||
# /v1/chat dispatcher reads this file at boot to populate its provider
|
||||
# table. Secrets (API keys) come from /etc/lakehouse/secrets.toml or
|
||||
# environment variables — NEVER inline a key here.
|
||||
#
|
||||
# Adding a new provider:
|
||||
# 1. New [[provider]] block with name, base_url, auth, default_model
|
||||
# 2. Matching adapter at crates/aibridge/src/providers/<name>.rs
|
||||
# implementing the ProviderAdapter trait (chat + embed + unload)
|
||||
# 3. Route arm in crates/gateway/src/v1/mod.rs matching on `name`
|
||||
# 4. Model-prefix routing hint in resolve_provider() if the provider
|
||||
# uses an "<name>/..." model prefix (e.g. "openrouter/...")
|
||||
|
||||
[[provider]]
|
||||
name = "ollama"
|
||||
base_url = "http://localhost:3200"
|
||||
auth = "none"
|
||||
default_model = "qwen3.5:latest"
|
||||
# Hot-path local inference. No bearer needed — Python sidecar on
|
||||
# localhost handles the Ollama API. Model names are bare
|
||||
# (e.g. "qwen3.5:latest", not "ollama/qwen3.5:latest").
|
||||
|
||||
[[provider]]
|
||||
name = "ollama_cloud"
|
||||
base_url = "https://ollama.com"
|
||||
auth = "bearer"
|
||||
auth_env = "OLLAMA_CLOUD_KEY"
|
||||
default_model = "gpt-oss:120b"
|
||||
# Cloud-tier Ollama. Key resolved from OLLAMA_CLOUD_KEY env at gateway
|
||||
# boot. Model-prefix routing: "cloud/<model>" auto-routes here
|
||||
# (see gateway::v1::resolve_provider).
|
||||
|
||||
[[provider]]
|
||||
name = "openrouter"
|
||||
base_url = "https://openrouter.ai/api/v1"
|
||||
auth = "bearer"
|
||||
auth_env = "OPENROUTER_API_KEY"
|
||||
auth_fallback_files = ["/home/profit/.env", "/root/llm_team_config.json"]
|
||||
default_model = "openai/gpt-oss-120b:free"
|
||||
# Multi-provider gateway. Covers Anthropic, Google, OpenAI, MiniMax,
|
||||
# Qwen, Gemma, etc. Key resolved via crates/gateway/src/v1/openrouter.rs
|
||||
# resolve_openrouter_key() — env first, then fallback files.
|
||||
# Model-prefix routing: "openrouter/<vendor>/<model>" auto-routes here,
|
||||
# prefix stripped before upstream call.
|
||||
|
||||
# Planned (Phase 40 long-horizon — adapters not yet shipped):
|
||||
#
|
||||
# [[provider]]
|
||||
# name = "gemini"
|
||||
# base_url = "https://generativelanguage.googleapis.com/v1beta"
|
||||
# auth = "api_key_query"
|
||||
# auth_env = "GEMINI_API_KEY"
|
||||
# default_model = "gemini-2.0-flash"
|
||||
#
|
||||
# [[provider]]
|
||||
# name = "claude"
|
||||
# base_url = "https://api.anthropic.com/v1"
|
||||
# auth = "x_api_key"
|
||||
# auth_env = "ANTHROPIC_API_KEY"
|
||||
# default_model = "claude-3-5-sonnet-latest"
|
||||
@@ -0,0 +1,61 @@
|
||||
# Phase 40: Routing Engine Configuration
|
||||
#
|
||||
# Human-editable rules for model → provider routing.
|
||||
# Matching order: first match wins.
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "gpt-4*"
|
||||
provider = "openrouter"
|
||||
max_tokens = 4096
|
||||
temperature = 0.7
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "claude*"
|
||||
provider = "claude"
|
||||
max_tokens = 4096
|
||||
temperature = 0.7
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "gemini*"
|
||||
provider = "gemini"
|
||||
max_tokens = 8192
|
||||
temperature = 0.9
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "qwen3.5*"
|
||||
provider = "ollama"
|
||||
max_tokens = 4096
|
||||
temperature = 0.3
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "qwen3*"
|
||||
provider = "ollama"
|
||||
max_tokens = 2048
|
||||
temperature = 0.3
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "gpt-oss*"
|
||||
provider = "ollama"
|
||||
temperature = 0.1
|
||||
|
||||
[[rule]]
|
||||
model_pattern = "*"
|
||||
provider = "ollama"
|
||||
temperature = 0.5
|
||||
|
||||
# Fallback chain: if primary fails, try these in order
|
||||
fallback = ["ollama", "openrouter"]
|
||||
|
||||
# Cost gating (tokens = cents per 1M)
|
||||
[cost]
|
||||
ollama = 0
|
||||
openrouter = 15
|
||||
claude = 15
|
||||
gemini = 0
|
||||
|
||||
# Daily budget per provider (cents)
|
||||
[daily_budget]
|
||||
ollama = 0
|
||||
openrouter = 1000
|
||||
claude = 500
|
||||
gemini = 0
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "aibridge"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
async-trait = "0.1"
|
||||
@@ -0,0 +1,188 @@
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
|
||||
/// HTTP client for the Python AI sidecar.
|
||||
#[derive(Clone)]
|
||||
pub struct AiClient {
|
||||
client: Client,
|
||||
base_url: String,
|
||||
}
|
||||
|
||||
// -- Request/Response types --
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct EmbedRequest {
|
||||
pub texts: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone)]
|
||||
pub struct EmbedResponse {
|
||||
pub embeddings: Vec<Vec<f64>>,
|
||||
pub model: String,
|
||||
pub dimensions: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateRequest {
|
||||
pub prompt: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
/// Phase 21 — per-call opt-out of hidden reasoning. Thinking models
|
||||
/// (qwen3.5, gpt-oss, etc) burn tokens on reasoning before the
|
||||
/// visible response starts; setting this to `false` on hot-path
|
||||
/// JSON emitters avoids empty returns when the budget is tight.
|
||||
/// Sidecar forwards this to Ollama's `think` parameter; if the
|
||||
/// sidecar drops an unknown field the request still succeeds.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub think: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone)]
|
||||
pub struct GenerateResponse {
|
||||
pub text: String,
|
||||
pub model: String,
|
||||
pub tokens_evaluated: Option<u64>,
|
||||
pub tokens_generated: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct RerankRequest {
|
||||
pub query: String,
|
||||
pub documents: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone)]
|
||||
pub struct ScoredDocument {
|
||||
pub index: usize,
|
||||
pub text: String,
|
||||
pub score: f64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone)]
|
||||
pub struct RerankResponse {
|
||||
pub results: Vec<ScoredDocument>,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl AiClient {
|
||||
pub fn new(base_url: &str) -> Self {
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()
|
||||
.expect("failed to build HTTP client");
|
||||
Self {
|
||||
client,
|
||||
base_url: base_url.trim_end_matches('/').to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn health(&self) -> Result<serde_json::Value, String> {
|
||||
let resp = self.client
|
||||
.get(format!("{}/health", self.base_url))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("sidecar unreachable: {e}"))?;
|
||||
resp.json().await.map_err(|e| format!("invalid response: {e}"))
|
||||
}
|
||||
|
||||
pub async fn embed(&self, req: EmbedRequest) -> Result<EmbedResponse, String> {
|
||||
let resp = self.client
|
||||
.post(format!("{}/embed", self.base_url))
|
||||
.json(&req)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("embed request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("embed error ({}): {text}", text.len()));
|
||||
}
|
||||
resp.json().await.map_err(|e| format!("embed parse error: {e}"))
|
||||
}
|
||||
|
||||
pub async fn generate(&self, req: GenerateRequest) -> Result<GenerateResponse, String> {
|
||||
let resp = self.client
|
||||
.post(format!("{}/generate", self.base_url))
|
||||
.json(&req)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("generate request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("generate error: {text}"));
|
||||
}
|
||||
resp.json().await.map_err(|e| format!("generate parse error: {e}"))
|
||||
}
|
||||
|
||||
pub async fn rerank(&self, req: RerankRequest) -> Result<RerankResponse, String> {
|
||||
let resp = self.client
|
||||
.post(format!("{}/rerank", self.base_url))
|
||||
.json(&req)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("rerank request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("rerank error: {text}"));
|
||||
}
|
||||
resp.json().await.map_err(|e| format!("rerank parse error: {e}"))
|
||||
}
|
||||
|
||||
/// Force Ollama to unload the named model from VRAM (keep_alive=0).
|
||||
/// Used for predictable profile swaps — without this, Ollama holds a
|
||||
/// model for its configured TTL (default 5min) and the previous
|
||||
/// profile's model can linger in VRAM next to the new one.
|
||||
pub async fn unload_model(&self, model: &str) -> Result<serde_json::Value, String> {
|
||||
let resp = self.client
|
||||
.post(format!("{}/admin/unload", self.base_url))
|
||||
.json(&serde_json::json!({ "model": model }))
|
||||
.send().await
|
||||
.map_err(|e| format!("unload request failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("unload error: {text}"));
|
||||
}
|
||||
resp.json().await.map_err(|e| format!("unload parse error: {e}"))
|
||||
}
|
||||
|
||||
/// Ask Ollama to load the named model into VRAM proactively. Makes
|
||||
/// the first real request after profile activation fast (no cold-load
|
||||
/// latency).
|
||||
pub async fn preload_model(&self, model: &str) -> Result<serde_json::Value, String> {
|
||||
let resp = self.client
|
||||
.post(format!("{}/admin/preload", self.base_url))
|
||||
.json(&serde_json::json!({ "model": model }))
|
||||
.send().await
|
||||
.map_err(|e| format!("preload request failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("preload error: {text}"));
|
||||
}
|
||||
resp.json().await.map_err(|e| format!("preload parse error: {e}"))
|
||||
}
|
||||
|
||||
/// GPU + loaded-model snapshot from the sidecar. Combines nvidia-smi
|
||||
/// output (if available) with Ollama's /api/ps.
|
||||
pub async fn vram_snapshot(&self) -> Result<serde_json::Value, String> {
|
||||
let resp = self.client
|
||||
.get(format!("{}/admin/vram", self.base_url))
|
||||
.send().await
|
||||
.map_err(|e| format!("vram request failed: {e}"))?;
|
||||
resp.json().await.map_err(|e| format!("vram parse error: {e}"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
//! Phase 21 — context-budget accounting for model calls.
|
||||
//!
|
||||
//! Ports `assertContextBudget` + `estimateTokens` + `CONTEXT_WINDOWS`
|
||||
//! from `tests/multi-agent/agent.ts` so Rust-side callers (gateway
|
||||
//! tool surfaces, future Rust agents) get the same loud-fail behavior
|
||||
//! on window overflow instead of silent truncation.
|
||||
//!
|
||||
//! The token estimator is deliberately the same chars/4 heuristic as
|
||||
//! the TS side. It's biased ~15% safe — pessimistic on English, correct
|
||||
//! within a factor of 2 on code. Swap to a provider tokenizer only when
|
||||
//! the estimator drives a decision (we're nowhere near that yet).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
// `estimate_tokens` moved to `shared::model_matrix::ModelMatrix::estimate_tokens`
|
||||
// (cdc24d8). All callers migrated; the deprecated wrapper that stood in its
|
||||
// place has been removed since it had zero external consumers.
|
||||
|
||||
/// Phase 21 — per-model context windows, mirroring the TS table in
|
||||
/// `tests/multi-agent/agent.ts`. Anchored on each model's documented
|
||||
/// max; unknown models fall back to `DEFAULT_CONTEXT_WINDOW`.
|
||||
pub const DEFAULT_CONTEXT_WINDOW: usize = 32_768;
|
||||
pub const DEFAULT_SAFETY_MARGIN: usize = 2_000;
|
||||
pub const DEFAULT_MAX_TOKENS: usize = 800;
|
||||
|
||||
fn known_windows() -> &'static HashMap<&'static str, usize> {
|
||||
static TABLE: OnceLock<HashMap<&'static str, usize>> = OnceLock::new();
|
||||
TABLE.get_or_init(|| {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("mistral:latest", 32_768);
|
||||
m.insert("qwen2.5:latest", 32_768);
|
||||
m.insert("qwen3:latest", 40_960);
|
||||
m.insert("qwen3.5:latest", 262_144);
|
||||
m.insert("qwen3-embedding", 32_768);
|
||||
m.insert("nomic-embed-text-v2-moe", 2_048);
|
||||
m.insert("gpt-oss:20b", 131_072);
|
||||
m.insert("gpt-oss:120b", 131_072);
|
||||
m.insert("qwen3.5:397b", 131_072);
|
||||
m.insert("kimi-k2-thinking", 200_000);
|
||||
m.insert("kimi-k2.6", 200_000);
|
||||
m.insert("kimi-k2:1t", 1_048_576);
|
||||
m.insert("deepseek-v3.1:671b", 131_072);
|
||||
m.insert("glm-4.7", 131_072);
|
||||
m
|
||||
})
|
||||
}
|
||||
|
||||
pub fn context_window_for(model: &str) -> usize {
|
||||
known_windows().get(model).copied().unwrap_or(DEFAULT_CONTEXT_WINDOW)
|
||||
}
|
||||
|
||||
/// Result of a budget check — exposes the numbers so callers can log
|
||||
/// how much headroom remains without re-running the estimator.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct BudgetCheck {
|
||||
pub estimated: usize,
|
||||
pub window: usize,
|
||||
pub remaining: i64,
|
||||
}
|
||||
|
||||
/// Inputs to `assert_context_budget`. `bypass` exists for call sites
|
||||
/// that handle their own overflow (continuation's second pass already
|
||||
/// counted the partial; T5 gatekeeper prompts have a separate policy).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BudgetOpts<'a> {
|
||||
pub system: Option<&'a str>,
|
||||
pub max_tokens: Option<usize>,
|
||||
pub safety_margin: Option<usize>,
|
||||
pub bypass: bool,
|
||||
}
|
||||
|
||||
/// Phase 21's loud-fail primitive. Returns a `BudgetCheck` on success
|
||||
/// and the same struct plus over-by count on failure. The whole point
|
||||
/// is to stop silent truncation — callers that expect overflow should
|
||||
/// chunk BEFORE calling or set `bypass: true`.
|
||||
pub fn assert_context_budget(
|
||||
model: &str,
|
||||
prompt: &str,
|
||||
opts: BudgetOpts,
|
||||
) -> Result<BudgetCheck, (BudgetCheck, usize)> {
|
||||
let window = context_window_for(model);
|
||||
let safety = opts.safety_margin.unwrap_or(DEFAULT_SAFETY_MARGIN);
|
||||
let max_tokens = opts.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS);
|
||||
let sys_tokens = opts.system.map(shared::model_matrix::ModelMatrix::estimate_tokens).unwrap_or(0);
|
||||
let estimated = shared::model_matrix::ModelMatrix::estimate_tokens(prompt) + sys_tokens + max_tokens;
|
||||
let remaining = window as i64 - estimated as i64 - safety as i64;
|
||||
let check = BudgetCheck { estimated, window, remaining };
|
||||
if remaining < 0 && !opts.bypass {
|
||||
return Err((check, (-remaining) as usize));
|
||||
}
|
||||
Ok(check)
|
||||
}
|
||||
|
||||
/// Convenience — format an overflow error the same way the TS side
|
||||
/// does. Exposed so downstream crates render consistent messages.
|
||||
pub fn overflow_message(model: &str, check: &BudgetCheck, over_by: usize, safety: usize) -> String {
|
||||
format!(
|
||||
"context overflow: model={} est={}t window={}t safety={}t over={}t. \
|
||||
Chunk the prompt (see config/models.json overflow_policies) or set \
|
||||
bypass:true if you know the risk.",
|
||||
model, check.estimated, check.window, safety, over_by,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Deprecated-function behavior is now canonically tested in
|
||||
// crates/shared/src/model_matrix.rs. This test was the legacy
|
||||
// pin that preceded the migration; delete when the deprecated
|
||||
// wrapper itself goes (see the #[deprecated] attribute).
|
||||
|
||||
#[test]
|
||||
fn context_window_known_and_fallback() {
|
||||
assert_eq!(context_window_for("qwen3.5:latest"), 262_144);
|
||||
assert_eq!(context_window_for("kimi-k2:1t"), 1_048_576);
|
||||
assert_eq!(context_window_for("some-unreleased-model"), DEFAULT_CONTEXT_WINDOW);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_passes_well_under_window() {
|
||||
let check = assert_context_budget(
|
||||
"qwen3:latest",
|
||||
&"x".repeat(4_000), // ~1000 tokens
|
||||
BudgetOpts { max_tokens: Some(500), ..Default::default() },
|
||||
).expect("well under 40K window");
|
||||
assert!(check.remaining > 30_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_fails_when_prompt_overflows_window() {
|
||||
let huge = "x".repeat(200_000); // ~50K tokens, over qwen3's 40K
|
||||
let err = assert_context_budget(
|
||||
"qwen3:latest",
|
||||
&huge,
|
||||
BudgetOpts::default(),
|
||||
).expect_err("should overflow qwen3's 40K window");
|
||||
assert!(err.1 > 0, "over_by must be positive");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_bypass_returns_ok_even_over() {
|
||||
let huge = "x".repeat(200_000);
|
||||
let check = assert_context_budget(
|
||||
"qwen3:latest",
|
||||
&huge,
|
||||
BudgetOpts { bypass: true, ..Default::default() },
|
||||
).expect("bypass must suppress the error");
|
||||
assert!(check.remaining < 0, "check still reports negative remaining");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_counts_system_prompt() {
|
||||
// 10K-char system prompt → ~2500 tokens. With a big max_tokens
|
||||
// this should push us closer to the window.
|
||||
let sys = "s".repeat(10_000);
|
||||
let prompt = "p".repeat(4_000);
|
||||
let with_sys = assert_context_budget(
|
||||
"qwen3:latest",
|
||||
&prompt,
|
||||
BudgetOpts {
|
||||
system: Some(&sys),
|
||||
max_tokens: Some(500),
|
||||
..Default::default()
|
||||
},
|
||||
).unwrap();
|
||||
let without_sys = assert_context_budget(
|
||||
"qwen3:latest",
|
||||
&prompt,
|
||||
BudgetOpts { max_tokens: Some(500), ..Default::default() },
|
||||
).unwrap();
|
||||
assert!(with_sys.estimated > without_sys.estimated,
|
||||
"system prompt should raise estimate");
|
||||
assert_eq!(with_sys.estimated - without_sys.estimated, shared::model_matrix::ModelMatrix::estimate_tokens(&sys));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overflow_message_includes_numbers() {
|
||||
let check = BudgetCheck { estimated: 42_000, window: 40_960, remaining: -1_040 };
|
||||
let msg = overflow_message("qwen3:latest", &check, 3_040, 2_000);
|
||||
assert!(msg.contains("qwen3:latest"));
|
||||
assert!(msg.contains("42000t"));
|
||||
assert!(msg.contains("40960t"));
|
||||
assert!(msg.contains("3040t"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
//! Phase 21 — OUTPUT-overflow handler. Ports `generateContinuable`
|
||||
//! from `tests/multi-agent/agent.ts`.
|
||||
//!
|
||||
//! Two failure modes to repair:
|
||||
//!
|
||||
//! * EMPTY response — thinking model ate the entire budget on hidden
|
||||
//! reasoning before emitting a token. Fix: retry the original prompt
|
||||
//! with 2× the budget, geometric up to `BUDGET_CAP`.
|
||||
//!
|
||||
//! * TRUNCATED non-empty — model got most of the way but hit
|
||||
//! max_tokens before closing the structure. Fix: continue with the
|
||||
//! partial response in the prompt as scratchpad, so the model knows
|
||||
//! where to pick up without restarting.
|
||||
//!
|
||||
//! `TextGenerator` abstracts the sidecar so tests can inject canned
|
||||
//! responses without a live Ollama.
|
||||
|
||||
use std::future::Future;
|
||||
|
||||
use crate::client::{AiClient, GenerateRequest, GenerateResponse};
|
||||
|
||||
/// Shape classifier for `is_structurally_complete`. JSON responses
|
||||
/// must parse; text responses just need to be non-empty.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ResponseShape {
|
||||
Json,
|
||||
Text,
|
||||
}
|
||||
|
||||
/// Trait that `generate_continuable` + `generate_tree_split` call. The
|
||||
/// real implementation forwards to `AiClient::generate`; tests supply a
|
||||
/// mock with a scripted sequence of responses.
|
||||
pub trait TextGenerator: Send + Sync {
|
||||
fn generate_text(
|
||||
&self,
|
||||
req: GenerateRequest,
|
||||
) -> impl Future<Output = Result<GenerateResponse, String>> + Send;
|
||||
}
|
||||
|
||||
impl TextGenerator for AiClient {
|
||||
fn generate_text(
|
||||
&self,
|
||||
req: GenerateRequest,
|
||||
) -> impl Future<Output = Result<GenerateResponse, String>> + Send {
|
||||
self.generate(req)
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip a surrounding ```json``` fence if present. Leaves the inner
|
||||
/// content alone otherwise. Returns a slice of `s`.
|
||||
fn strip_json_fence(s: &str) -> &str {
|
||||
let t = s.trim();
|
||||
if let Some(rest) = t.strip_prefix("```json") {
|
||||
rest.trim_start_matches('\n').strip_suffix("```").unwrap_or(rest).trim()
|
||||
} else if let Some(rest) = t.strip_prefix("```") {
|
||||
rest.trim_start_matches('\n').strip_suffix("```").unwrap_or(rest).trim()
|
||||
} else {
|
||||
t
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of the TS brace-balance + JSON.parse check. Returns true when
|
||||
/// the outermost `{...}` block is balanced and parses. Text shape is
|
||||
/// satisfied by any non-empty, non-whitespace payload.
|
||||
pub fn is_structurally_complete(text: &str, shape: ResponseShape) -> bool {
|
||||
if text.trim().is_empty() { return false; }
|
||||
if shape == ResponseShape::Text { return true; }
|
||||
|
||||
let s = strip_json_fence(text);
|
||||
let Some(start) = s.find('{') else { return false };
|
||||
let Some(end) = s.rfind('}') else { return false };
|
||||
if end <= start { return false; }
|
||||
let slice = &s[start..=end];
|
||||
|
||||
// Balance check — cheaper than parse, catches truncated nests.
|
||||
// String state tracked because `{` inside a string doesn't count.
|
||||
let mut depth: i32 = 0;
|
||||
let mut in_str = false;
|
||||
let mut esc = false;
|
||||
for c in slice.chars() {
|
||||
if esc { esc = false; continue; }
|
||||
if c == '\\' { esc = true; continue; }
|
||||
if c == '"' { in_str = !in_str; continue; }
|
||||
if in_str { continue; }
|
||||
if c == '{' { depth += 1; }
|
||||
else if c == '}' {
|
||||
depth -= 1;
|
||||
if depth < 0 { return false; }
|
||||
}
|
||||
}
|
||||
if depth != 0 { return false; }
|
||||
// Parse check is the tie-breaker — balanced but invalid JSON (e.g.
|
||||
// trailing comma before `}`) shouldn't count as complete.
|
||||
serde_json::from_str::<serde_json::Value>(slice).is_ok()
|
||||
}
|
||||
|
||||
/// Knobs for `generate_continuable`. All optional with sensible
|
||||
/// defaults that match the TS version.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ContinuableOpts {
|
||||
pub model: String,
|
||||
pub max_tokens: Option<u32>,
|
||||
pub temperature: Option<f64>,
|
||||
pub system: Option<String>,
|
||||
pub shape: ResponseShape,
|
||||
pub max_continuations: usize,
|
||||
pub think: Option<bool>,
|
||||
/// Geometric-backoff ceiling for the empty-response retry path.
|
||||
/// Matches TS's `budgetCap = 8000`.
|
||||
pub budget_cap: u32,
|
||||
/// Maximum empty-response retries before giving up. Matches TS's
|
||||
/// hardcoded `retry < 3`.
|
||||
pub max_empty_retries: usize,
|
||||
}
|
||||
|
||||
impl ContinuableOpts {
|
||||
pub fn new(model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
model: model.into(),
|
||||
max_tokens: None,
|
||||
temperature: None,
|
||||
system: None,
|
||||
shape: ResponseShape::Json,
|
||||
max_continuations: 3,
|
||||
think: None,
|
||||
budget_cap: 8_000,
|
||||
max_empty_retries: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of a `generate_continuable` call. Carries the combined
|
||||
/// text plus diagnostic counters so observability downstream can
|
||||
/// report "how many continuations did that query cost".
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ContinuableOutcome {
|
||||
pub text: String,
|
||||
pub empty_retries: usize,
|
||||
pub continuations: usize,
|
||||
pub final_complete: bool,
|
||||
/// Sum of `prompt_tokens` across every generator call made to
|
||||
/// produce this outcome — including empty retries and continuations.
|
||||
/// Lets callers (gateway execution loop, observability) stamp
|
||||
/// accurate per-task usage without second-guessing the retry fan-out.
|
||||
pub prompt_tokens: u32,
|
||||
/// Sum of `completion_tokens` across every generator call.
|
||||
pub completion_tokens: u32,
|
||||
/// Total number of generator calls. `1 + empty_retries +
|
||||
/// continuations` in the normal case; the field is explicit so
|
||||
/// callers don't have to re-derive it.
|
||||
pub calls: u32,
|
||||
}
|
||||
|
||||
fn make_request(opts: &ContinuableOpts, prompt: String, current_max: u32) -> GenerateRequest {
|
||||
GenerateRequest {
|
||||
prompt,
|
||||
model: Some(opts.model.clone()),
|
||||
system: opts.system.clone(),
|
||||
temperature: opts.temperature,
|
||||
max_tokens: Some(current_max),
|
||||
think: opts.think,
|
||||
}
|
||||
}
|
||||
|
||||
fn continuation_prompt(original: &str, partial: &str) -> String {
|
||||
format!(
|
||||
"{original}\n\n\
|
||||
PARTIAL RESPONSE SO FAR (continue from here — do NOT restart, \
|
||||
do NOT repeat what's already there, emit ONLY the remaining \
|
||||
tokens to close the structure):\n{partial}"
|
||||
)
|
||||
}
|
||||
|
||||
/// Phase 21 — output-overflow safe generate. See module docs for the
|
||||
/// two failure modes repaired. On final-still-incomplete, returns the
|
||||
/// combined text with `final_complete: false` so the caller's parser
|
||||
/// can throw with the raw text for forensics rather than silently
|
||||
/// truncating.
|
||||
pub async fn generate_continuable<G: TextGenerator>(
|
||||
generator: &G,
|
||||
prompt: &str,
|
||||
opts: &ContinuableOpts,
|
||||
) -> Result<ContinuableOutcome, String> {
|
||||
let initial_max = opts.max_tokens.unwrap_or(800);
|
||||
let mut current_max = initial_max;
|
||||
let mut combined = String::new();
|
||||
let mut empty_retries = 0usize;
|
||||
let mut continuations = 0usize;
|
||||
let mut prompt_tokens: u32 = 0;
|
||||
let mut completion_tokens: u32 = 0;
|
||||
let mut calls: u32 = 0;
|
||||
|
||||
// Phase 21(a) — empty-response backoff loop.
|
||||
for retry in 0..opts.max_empty_retries {
|
||||
let req = make_request(opts, prompt.to_string(), current_max);
|
||||
let resp = generator.generate_text(req).await?;
|
||||
calls += 1;
|
||||
prompt_tokens = prompt_tokens.saturating_add(resp.tokens_evaluated.unwrap_or(0) as u32);
|
||||
completion_tokens = completion_tokens.saturating_add(resp.tokens_generated.unwrap_or(0) as u32);
|
||||
if !resp.text.trim().is_empty() {
|
||||
combined = resp.text;
|
||||
break;
|
||||
}
|
||||
empty_retries = retry + 1;
|
||||
current_max = (current_max.saturating_mul(2)).min(opts.budget_cap);
|
||||
}
|
||||
|
||||
// Phase 21(b) — structural-completion continuation loop.
|
||||
for _ in 0..opts.max_continuations {
|
||||
if is_structurally_complete(&combined, opts.shape) {
|
||||
return Ok(ContinuableOutcome {
|
||||
text: combined,
|
||||
empty_retries,
|
||||
continuations,
|
||||
final_complete: true,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
calls,
|
||||
});
|
||||
}
|
||||
if combined.trim().is_empty() {
|
||||
// Nothing to continue from — continuing "" is identical to
|
||||
// the initial call and would loop.
|
||||
break;
|
||||
}
|
||||
let cont_prompt = continuation_prompt(prompt, &combined);
|
||||
let req = make_request(opts, cont_prompt, current_max.min(opts.budget_cap));
|
||||
let resp = generator.generate_text(req).await?;
|
||||
calls += 1;
|
||||
prompt_tokens = prompt_tokens.saturating_add(resp.tokens_evaluated.unwrap_or(0) as u32);
|
||||
completion_tokens = completion_tokens.saturating_add(resp.tokens_generated.unwrap_or(0) as u32);
|
||||
combined.push_str(&resp.text);
|
||||
continuations += 1;
|
||||
}
|
||||
|
||||
let final_complete = is_structurally_complete(&combined, opts.shape);
|
||||
Ok(ContinuableOutcome {
|
||||
text: combined,
|
||||
empty_retries,
|
||||
continuations,
|
||||
final_complete,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
calls,
|
||||
})
|
||||
}
|
||||
|
||||
/// Scripted generator for unit tests. Returns responses from `script`
|
||||
/// in order; extra calls reuse the last entry so tests don't have to
|
||||
/// count past what they actually assert on.
|
||||
#[cfg(test)]
|
||||
pub struct ScriptedGenerator {
|
||||
script: Vec<Result<String, String>>,
|
||||
calls: std::sync::Arc<std::sync::Mutex<Vec<GenerateRequest>>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl ScriptedGenerator {
|
||||
pub fn new<I>(script: I) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = Result<String, String>>,
|
||||
{
|
||||
Self {
|
||||
script: script.into_iter().collect(),
|
||||
calls: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calls(&self) -> Vec<GenerateRequest> {
|
||||
self.calls.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
pub fn call_count(&self) -> usize {
|
||||
self.calls.lock().unwrap().len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl TextGenerator for ScriptedGenerator {
|
||||
fn generate_text(
|
||||
&self,
|
||||
req: GenerateRequest,
|
||||
) -> impl Future<Output = Result<GenerateResponse, String>> + Send {
|
||||
let i = {
|
||||
let mut calls = self.calls.lock().unwrap();
|
||||
let i = calls.len();
|
||||
calls.push(req.clone());
|
||||
i
|
||||
};
|
||||
let model = req.model.clone().unwrap_or_default();
|
||||
let entry = self.script.get(i)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.script.last().cloned().unwrap_or(Ok(String::new())));
|
||||
async move {
|
||||
entry.map(|text| GenerateResponse {
|
||||
text,
|
||||
model,
|
||||
tokens_evaluated: None,
|
||||
tokens_generated: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn structural_complete_rejects_empty_and_text_mismatch() {
|
||||
assert!(!is_structurally_complete("", ResponseShape::Text));
|
||||
assert!(!is_structurally_complete(" ", ResponseShape::Text));
|
||||
assert!(is_structurally_complete("any content", ResponseShape::Text));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_complete_handles_balanced_json() {
|
||||
assert!(is_structurally_complete(r#"{"a": 1}"#, ResponseShape::Json));
|
||||
assert!(is_structurally_complete(
|
||||
r#"```json
|
||||
{"a": 1, "b": [1, 2, 3]}
|
||||
```"#,
|
||||
ResponseShape::Json,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_complete_rejects_truncated_json() {
|
||||
assert!(!is_structurally_complete(r#"{"a": 1"#, ResponseShape::Json));
|
||||
assert!(!is_structurally_complete(r#"{"a": {"b": 1"#, ResponseShape::Json));
|
||||
// Trailing comma — balanced but unparseable
|
||||
assert!(!is_structurally_complete(r#"{"a": 1,}"#, ResponseShape::Json));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_complete_ignores_braces_inside_strings() {
|
||||
assert!(is_structurally_complete(r#"{"s": "has } inside"}"#, ResponseShape::Json));
|
||||
assert!(is_structurally_complete(r#"{"s": "escaped \" quote"}"#, ResponseShape::Json));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_returns_first_response_when_complete() {
|
||||
let generator = ScriptedGenerator::new(vec![Ok(r#"{"ok": true}"#.to_string())]);
|
||||
let opts = ContinuableOpts::new("qwen3:latest");
|
||||
let out = generate_continuable(&generator, "test", &opts).await.unwrap();
|
||||
assert!(out.final_complete);
|
||||
assert_eq!(out.empty_retries, 0);
|
||||
assert_eq!(out.continuations, 0);
|
||||
assert_eq!(generator.call_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_retries_on_empty_with_doubled_budget() {
|
||||
// Two empties, then a good response. Third call should see 4×
|
||||
// the initial budget (2× twice).
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("".to_string()),
|
||||
Ok("".to_string()),
|
||||
Ok(r#"{"ok": true}"#.to_string()),
|
||||
]);
|
||||
let mut opts = ContinuableOpts::new("qwen3:latest");
|
||||
opts.max_tokens = Some(100);
|
||||
let out = generate_continuable(&generator, "test", &opts).await.unwrap();
|
||||
assert!(out.final_complete);
|
||||
assert_eq!(out.empty_retries, 2);
|
||||
|
||||
let calls = generator.calls();
|
||||
assert_eq!(calls.len(), 3);
|
||||
assert_eq!(calls[0].max_tokens, Some(100));
|
||||
assert_eq!(calls[1].max_tokens, Some(200));
|
||||
assert_eq!(calls[2].max_tokens, Some(400));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_caps_budget_at_budget_cap() {
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("".to_string()),
|
||||
Ok("".to_string()),
|
||||
Ok(r#"{"ok":1}"#.to_string()),
|
||||
]);
|
||||
let mut opts = ContinuableOpts::new("qwen3:latest");
|
||||
opts.max_tokens = Some(5_000);
|
||||
opts.budget_cap = 7_000;
|
||||
let out = generate_continuable(&generator, "test", &opts).await.unwrap();
|
||||
assert!(out.final_complete);
|
||||
let calls = generator.calls();
|
||||
// 5000 → doubled would be 10000; cap pulls it to 7000.
|
||||
assert_eq!(calls[1].max_tokens, Some(7_000));
|
||||
assert_eq!(calls[2].max_tokens, Some(7_000));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_glues_truncated_response() {
|
||||
// First call returns balanced-open `{...`; continuation closes
|
||||
// it with `...}`. Combined must parse.
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok(r#"{"fills": [{"candidate_id": "C-001""#.to_string()),
|
||||
Ok(r#", "name": "Alice"}]}"#.to_string()),
|
||||
]);
|
||||
let opts = ContinuableOpts::new("qwen3:latest");
|
||||
let out = generate_continuable(&generator, "ORIGINAL", &opts).await.unwrap();
|
||||
assert!(out.final_complete, "combined must parse: {}", out.text);
|
||||
assert_eq!(out.continuations, 1);
|
||||
|
||||
let calls = generator.calls();
|
||||
assert_eq!(calls.len(), 2);
|
||||
// Continuation prompt must contain the partial — that's the
|
||||
// scratchpad primitive J called out.
|
||||
let cont_prompt = &calls[1].prompt;
|
||||
assert!(cont_prompt.contains("ORIGINAL"),
|
||||
"continuation must include original prompt");
|
||||
assert!(cont_prompt.contains(r#"{"fills": [{"candidate_id": "C-001""#),
|
||||
"continuation must include partial");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_does_not_loop_on_persistent_empty() {
|
||||
// All three retries return empty; we must NOT then enter the
|
||||
// continuation loop with an empty partial (would burn 3 more
|
||||
// calls continuing from "").
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("".to_string()),
|
||||
Ok("".to_string()),
|
||||
Ok("".to_string()),
|
||||
]);
|
||||
let opts = ContinuableOpts::new("qwen3:latest");
|
||||
let out = generate_continuable(&generator, "test", &opts).await.unwrap();
|
||||
assert!(!out.final_complete);
|
||||
assert_eq!(out.empty_retries, 3);
|
||||
assert_eq!(out.continuations, 0, "must not continue from empty");
|
||||
assert_eq!(generator.call_count(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_returns_raw_on_exhausted_continuations() {
|
||||
// Three continuations that never complete — caller's parser
|
||||
// will throw. We must return the combined text so forensics
|
||||
// has the raw content, not a lossy truncation.
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok(r#"{"a": ["#.to_string()),
|
||||
Ok("1,".to_string()),
|
||||
Ok("2,".to_string()),
|
||||
Ok("3,".to_string()),
|
||||
]);
|
||||
let mut opts = ContinuableOpts::new("qwen3:latest");
|
||||
opts.max_continuations = 3;
|
||||
let out = generate_continuable(&generator, "test", &opts).await.unwrap();
|
||||
assert!(!out.final_complete);
|
||||
assert_eq!(out.continuations, 3);
|
||||
assert!(out.text.contains(r#"{"a": ["#));
|
||||
assert!(out.text.contains("3,"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuable_propagates_generator_errors() {
|
||||
let generator = ScriptedGenerator::new(vec![Err("sidecar 503".to_string())]);
|
||||
let opts = ContinuableOpts::new("qwen3:latest");
|
||||
let err = generate_continuable(&generator, "test", &opts).await.unwrap_err();
|
||||
assert!(err.contains("503"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
pub mod client;
|
||||
pub mod context;
|
||||
pub mod continuation;
|
||||
pub mod provider;
|
||||
pub mod providers;
|
||||
pub mod routing;
|
||||
pub mod service;
|
||||
pub mod tree_split;
|
||||
@@ -0,0 +1,39 @@
|
||||
use async_trait::async_trait;
|
||||
use crate::client::{GenerateRequest, GenerateResponse, EmbedRequest, EmbedResponse};
|
||||
|
||||
#[async_trait]
|
||||
pub trait ProviderAdapter: Send + Sync {
|
||||
/// Name for routing (ollama, openrouter, etc.)
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Chat completion — returns text, model, token counts
|
||||
async fn chat(&self, req: GenerateRequest) -> Result<GenerateResponse, String>;
|
||||
|
||||
/// Embeddings — returns vectors, model, dimensions
|
||||
async fn embed(&self, req: EmbedRequest) -> Result<EmbedResponse, String>;
|
||||
|
||||
/// Unload model from VRAM (optional, no-op if not supported)
|
||||
async fn unload(&self, _model: &str) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Health check
|
||||
async fn health(&self) -> Result<serde_json::Value, String>;
|
||||
}
|
||||
|
||||
/// Routing key extracted from model name.
|
||||
/// - "qwen3.5:latest" → "ollama"
|
||||
/// - "openrouter/anthropic/claude-3.5-sonnet" → "openrouter"
|
||||
/// - "gpt-4o" → "ollama" (default)
|
||||
pub fn provider_key(model: &str) -> &'static str {
|
||||
let lower = model.to_lowercase();
|
||||
if lower.starts_with("openrouter/") {
|
||||
"openrouter"
|
||||
} else if lower.starts_with("gemini") {
|
||||
"gemini"
|
||||
} else if lower.starts_with("claude") {
|
||||
"claude"
|
||||
} else {
|
||||
"ollama" // default: local Ollama
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod ollama;
|
||||
pub mod openrouter;
|
||||
@@ -0,0 +1,37 @@
|
||||
use async_trait::async_trait;
|
||||
use crate::client::{AiClient, GenerateRequest, GenerateResponse, EmbedRequest, EmbedResponse};
|
||||
use crate::provider::ProviderAdapter;
|
||||
|
||||
pub struct OllamaAdapter {
|
||||
client: AiClient,
|
||||
}
|
||||
|
||||
impl OllamaAdapter {
|
||||
pub fn new(client: AiClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAdapter for OllamaAdapter {
|
||||
fn name(&self) -> &str {
|
||||
"ollama"
|
||||
}
|
||||
|
||||
async fn chat(&self, req: GenerateRequest) -> Result<GenerateResponse, String> {
|
||||
self.client.generate(req).await
|
||||
}
|
||||
|
||||
async fn embed(&self, req: EmbedRequest) -> Result<EmbedResponse, String> {
|
||||
self.client.embed(req).await
|
||||
}
|
||||
|
||||
async fn unload(&self, model: &str) -> Result<(), String> {
|
||||
let _: serde_json::Value = self.client.unload_model(model).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn health(&self) -> Result<serde_json::Value, String> {
|
||||
self.client.health().await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
use async_trait::async_trait;
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::client::{GenerateRequest, GenerateResponse, EmbedRequest, EmbedResponse};
|
||||
use crate::provider::ProviderAdapter;
|
||||
|
||||
pub struct OpenRouterAdapter {
|
||||
client: Client,
|
||||
base_url: String,
|
||||
api_key: String,
|
||||
default_model: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct OpenRouterChatRequest {
|
||||
model: String,
|
||||
messages: Vec<OpenRouterMessage>,
|
||||
temperature: Option<f64>,
|
||||
max_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct OpenRouterMessage {
|
||||
role: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OpenRouterChatResponse {
|
||||
choices: Vec<OpenRouterChoice>,
|
||||
usage: OpenRouterUsage,
|
||||
model: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OpenRouterChoice {
|
||||
message: OpenRouterMessageOut,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
struct OpenRouterMessageOut {
|
||||
role: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
struct OpenRouterUsage {
|
||||
prompt_tokens: Option<u32>,
|
||||
completion_tokens: Option<u32>,
|
||||
total_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
impl OpenRouterAdapter {
|
||||
pub fn new(base_url: &str, api_key: String, default_model: &str) -> Self {
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(180))
|
||||
.build()
|
||||
.expect("failed to build HTTP client");
|
||||
Self {
|
||||
client,
|
||||
base_url: base_url.trim_end_matches('/').to_string(),
|
||||
api_key,
|
||||
default_model: default_model.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_model(&self, model: &str) -> String {
|
||||
// Strip "openrouter/" prefix if present
|
||||
let m = model.trim_start_matches("openrouter/");
|
||||
if m.is_empty() || m == model {
|
||||
self.default_model.clone()
|
||||
} else {
|
||||
m.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn to_openrouter_messages(req: &GenerateRequest) -> Vec<OpenRouterMessage> {
|
||||
let mut out = vec![];
|
||||
if let Some(sys) = &req.system {
|
||||
out.push(OpenRouterMessage { role: "system".into(), content: sys.clone() });
|
||||
}
|
||||
out.push(OpenRouterMessage {
|
||||
role: "user".into(),
|
||||
content: req.prompt.clone(),
|
||||
});
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAdapter for OpenRouterAdapter {
|
||||
fn name(&self) -> &str {
|
||||
"openrouter"
|
||||
}
|
||||
|
||||
async fn chat(&self, req: GenerateRequest) -> Result<GenerateResponse, String> {
|
||||
let model = self.chat_model(req.model.as_deref().unwrap_or(""));
|
||||
let or_req = OpenRouterChatRequest {
|
||||
model: model.clone(),
|
||||
messages: OpenRouterAdapter::to_openrouter_messages(&req),
|
||||
temperature: req.temperature,
|
||||
max_tokens: req.max_tokens,
|
||||
};
|
||||
|
||||
let resp = self.client
|
||||
.post(format!("{}/chat/completions", self.base_url))
|
||||
.header("Authorization", format!("Bearer {}", self.api_key))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&or_req)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("openrouter request failed: {e}"))?;
|
||||
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
|
||||
if !status.is_success() {
|
||||
return Err(format!("openrouter error ({}): {}", status, body));
|
||||
}
|
||||
|
||||
let or_resp: OpenRouterChatResponse = serde_json::from_str(&body)
|
||||
.map_err(|e| format!("openrouter parse error: {e}"))?;
|
||||
|
||||
let choice = or_resp.choices.into_iter().next()
|
||||
.ok_or("no completion choice returned")?;
|
||||
let usage = or_resp.usage;
|
||||
|
||||
Ok(GenerateResponse {
|
||||
text: choice.message.content,
|
||||
model: or_resp.model,
|
||||
tokens_evaluated: usage.prompt_tokens.map(|n| n as u64),
|
||||
tokens_generated: usage.completion_tokens.map(|n| n as u64),
|
||||
})
|
||||
}
|
||||
|
||||
async fn embed(&self, _req: EmbedRequest) -> Result<EmbedResponse, String> {
|
||||
Err("openrouter: embed not implemented".into())
|
||||
}
|
||||
|
||||
async fn health(&self) -> Result<serde_json::Value, String> {
|
||||
// OpenRouter doesn't have a dedicated health endpoint,
|
||||
// so we just return a healthy response if the client works.
|
||||
Ok(serde_json::json!({
|
||||
"status": "ok",
|
||||
"provider": "openrouter",
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct RoutingRule {
|
||||
pub model_pattern: String,
|
||||
pub provider: String,
|
||||
pub max_tokens: Option<u32>,
|
||||
pub temperature: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RoutingEngine {
|
||||
rules: Vec<RoutingRule>,
|
||||
fallback_chain: Vec<String>,
|
||||
}
|
||||
|
||||
impl RoutingEngine {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn with_rules(mut self, rules: Vec<RoutingRule>) -> Self {
|
||||
self.rules = rules;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_fallback(mut self, chain: Vec<String>) -> Self {
|
||||
self.fallback_chain = chain;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn route(&self, model: &str) -> RouteDecision {
|
||||
let lower = model.to_lowercase();
|
||||
|
||||
for rule in &self.rules {
|
||||
if glob_match(&rule.model_pattern.to_lowercase(), &lower) {
|
||||
return RouteDecision {
|
||||
provider: rule.provider.clone(),
|
||||
model: model.to_string(),
|
||||
max_tokens: rule.max_tokens,
|
||||
temperature: rule.temperature,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(first) = self.fallback_chain.first() {
|
||||
RouteDecision {
|
||||
provider: first.clone(),
|
||||
model: model.to_string(),
|
||||
max_tokens: None,
|
||||
temperature: None,
|
||||
}
|
||||
} else {
|
||||
RouteDecision {
|
||||
provider: "ollama".to_string(),
|
||||
model: model.to_string(),
|
||||
max_tokens: None,
|
||||
temperature: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RouteDecision {
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub max_tokens: Option<u32>,
|
||||
pub temperature: Option<f64>,
|
||||
}
|
||||
|
||||
fn glob_match(pattern: &str, name: &str) -> bool {
|
||||
if !pattern.contains('*') { return pattern == name; }
|
||||
let parts: Vec<&str> = pattern.split('*').collect();
|
||||
// Multi-* support: first must be prefix, last must be suffix, each
|
||||
// interior piece must appear in order. Fixes the iter-9 finding
|
||||
// where gpt-*-large* silently fell through to an exact-match path.
|
||||
// Also removes the dead `parts.len() == 1` branch that accessed
|
||||
// parts[1] and would panic if ever reached (unreachable today
|
||||
// since split('*') on a string containing '*' always yields ≥2).
|
||||
if !name.starts_with(parts[0]) || !name.ends_with(parts.last().unwrap()) { return false; }
|
||||
let mut cursor = parts[0].len();
|
||||
parts[1..parts.len() - 1].iter().all(|mid| {
|
||||
name[cursor..].find(mid).map(|pos| { cursor += pos + mid.len(); true }).unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
impl Default for RoutingRule {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
model_pattern: "*".to_string(),
|
||||
provider: "ollama".to_string(),
|
||||
max_tokens: None,
|
||||
temperature: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod glob_match_tests {
|
||||
use super::glob_match;
|
||||
|
||||
#[test] fn exact_match() { assert!(glob_match("gpt-oss:120b", "gpt-oss:120b")); }
|
||||
#[test] fn exact_mismatch() { assert!(!glob_match("a", "b")); }
|
||||
#[test] fn leading_wildcard() { assert!(glob_match("*:120b", "gpt-oss:120b")); }
|
||||
#[test] fn trailing_wildcard() { assert!(glob_match("gpt-oss:*", "gpt-oss:120b")); }
|
||||
#[test] fn bare_wildcard() { assert!(glob_match("*", "anything")); }
|
||||
#[test] fn multi_wildcard_in_order() { assert!(glob_match("gpt-*-oss-*", "gpt-4-oss-120b")); }
|
||||
#[test] fn multi_wildcard_wrong_order() { assert!(!glob_match("b*a*", "abba")); }
|
||||
#[test] fn multi_wildcard_panic_safety() {
|
||||
// Regression: earlier impl had an unreachable `parts.len() == 1`
|
||||
// branch that indexed parts[1] — would panic if ever hit. Now
|
||||
// the split('*') invariant guarantees ≥2 parts when * present,
|
||||
// and we handle all N-part cases explicitly.
|
||||
assert!(glob_match("a*b*c", "abc"));
|
||||
assert!(glob_match("a*b*c", "axxxbxxxc"));
|
||||
assert!(!glob_match("a*b*c", "xxxbxxx"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::State,
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
|
||||
use crate::client::{
|
||||
AiClient, EmbedRequest, GenerateRequest, RerankRequest,
|
||||
};
|
||||
|
||||
pub fn router(client: AiClient) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/embed", post(embed))
|
||||
.route("/generate", post(generate))
|
||||
.route("/rerank", post(rerank))
|
||||
.route("/vram", get(vram))
|
||||
.with_state(client)
|
||||
}
|
||||
|
||||
async fn vram(State(client): State<AiClient>) -> impl IntoResponse {
|
||||
match client.vram_snapshot().await {
|
||||
Ok(snap) => Ok(Json(snap)),
|
||||
Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn health(State(client): State<AiClient>) -> impl IntoResponse {
|
||||
match client.health().await {
|
||||
Ok(info) => Ok(Json(info)),
|
||||
Err(e) => Err((StatusCode::BAD_GATEWAY, format!("sidecar down: {e}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn embed(
|
||||
State(client): State<AiClient>,
|
||||
Json(req): Json<EmbedRequest>,
|
||||
) -> impl IntoResponse {
|
||||
tracing::info!("embedding {} texts", req.texts.len());
|
||||
match client.embed(req).await {
|
||||
Ok(resp) => Ok(Json(resp)),
|
||||
Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn generate(
|
||||
State(client): State<AiClient>,
|
||||
Json(req): Json<GenerateRequest>,
|
||||
) -> impl IntoResponse {
|
||||
tracing::info!("generating with prompt len={}", req.prompt.len());
|
||||
match client.generate(req).await {
|
||||
Ok(resp) => Ok(Json(resp)),
|
||||
Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn rerank(
|
||||
State(client): State<AiClient>,
|
||||
Json(req): Json<RerankRequest>,
|
||||
) -> impl IntoResponse {
|
||||
tracing::info!("reranking {} documents", req.documents.len());
|
||||
match client.rerank(req).await {
|
||||
Ok(resp) => Ok(Json(resp)),
|
||||
Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
//! Phase 21 — INPUT-overflow handler. Ports `generateTreeSplit` from
|
||||
//! `tests/multi-agent/agent.ts`.
|
||||
//!
|
||||
//! When the input corpus exceeds the model's window (200 playbooks
|
||||
//! pasted into a T4 strategic prompt, a long retrospective digest, a
|
||||
//! cross-corpus summarization), raising `max_tokens` doesn't help —
|
||||
//! the prompt itself is the problem. The answer is map-reduce:
|
||||
//!
|
||||
//! 1. Caller shards the input at semantic boundaries (records,
|
||||
//! paragraphs, playbook entries).
|
||||
//! 2. For each shard, build a map prompt that includes the running
|
||||
//! scratchpad and run it through `generate_continuable`.
|
||||
//! 3. Append the map output to the scratchpad (oldest-first
|
||||
//! truncation when it outgrows `scratchpad_budget`).
|
||||
//! 4. Build a reduce prompt from the final scratchpad and run it.
|
||||
//!
|
||||
//! Every shard prompt and the reduce prompt go through
|
||||
//! `assert_context_budget` first — if a single shard still overflows
|
||||
//! we bubble the error up rather than silently truncating. That's the
|
||||
//! whole point of Phase 21.
|
||||
|
||||
use crate::context::{assert_context_budget, BudgetOpts, overflow_message,
|
||||
DEFAULT_MAX_TOKENS, DEFAULT_SAFETY_MARGIN};
|
||||
use shared::model_matrix::ModelMatrix;
|
||||
use crate::continuation::{generate_continuable, ContinuableOpts, ResponseShape, TextGenerator};
|
||||
|
||||
/// Callback signatures — caller supplies closures that stitch the
|
||||
/// scratchpad into each shard's prompt and build the final reduce
|
||||
/// prompt. Kept as `Fn` (not `FnMut`) so the map loop can call them
|
||||
/// by reference.
|
||||
pub type MapPromptFn<'a> = dyn Fn(&str, &str) -> String + Send + Sync + 'a;
|
||||
pub type ReducePromptFn<'a> = dyn Fn(&str) -> String + Send + Sync + 'a;
|
||||
|
||||
/// Knobs for `generate_tree_split`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TreeSplitOpts {
|
||||
pub model: String,
|
||||
pub system: Option<String>,
|
||||
pub temperature: Option<f64>,
|
||||
/// max_tokens for map AND reduce (reduce defaults are usually
|
||||
/// higher; caller overrides for just reduce by calling through
|
||||
/// continuable directly if needed).
|
||||
pub max_tokens: Option<u32>,
|
||||
pub reduce_max_tokens: Option<u32>,
|
||||
pub think: Option<bool>,
|
||||
/// Soft ceiling on scratchpad size (estimated tokens). When it
|
||||
/// grows past this, the oldest shard digest gets dropped. Default
|
||||
/// 6000, matching the TS implementation.
|
||||
pub scratchpad_budget: usize,
|
||||
pub safety_margin: Option<usize>,
|
||||
}
|
||||
|
||||
impl TreeSplitOpts {
|
||||
pub fn new(model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
model: model.into(),
|
||||
system: None,
|
||||
temperature: None,
|
||||
max_tokens: None,
|
||||
reduce_max_tokens: None,
|
||||
think: None,
|
||||
scratchpad_budget: 6_000,
|
||||
safety_margin: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result — final reduce response plus the accumulated scratchpad so
|
||||
/// the caller can inspect what was kept vs truncated.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TreeSplitResult {
|
||||
pub response: String,
|
||||
pub scratchpad: String,
|
||||
pub shards_processed: usize,
|
||||
pub scratchpad_truncations: usize,
|
||||
pub total_continuations: usize,
|
||||
}
|
||||
|
||||
/// Drop shard-digest blocks from the head of `scratchpad` until its
|
||||
/// estimated-token count fits the budget. Digest blocks are delimited
|
||||
/// by `\n— shard N/M digest —\n` so we can find the first one and
|
||||
/// chop everything before its successor.
|
||||
fn truncate_scratchpad(scratchpad: &mut String, budget_tokens: usize) -> bool {
|
||||
if ModelMatrix::estimate_tokens(scratchpad) <= budget_tokens { return false; }
|
||||
// Find the second delimiter — everything before it gets dropped.
|
||||
const DELIM_PREFIX: &str = "\n— shard ";
|
||||
let mut cursor = 0;
|
||||
let mut truncated = false;
|
||||
while ModelMatrix::estimate_tokens(&scratchpad[cursor..]) > budget_tokens {
|
||||
// Skip past a leading delimiter (if we're sitting on one from
|
||||
// a previous iteration), then find the next.
|
||||
let search_from = cursor + if scratchpad[cursor..].starts_with(DELIM_PREFIX) {
|
||||
DELIM_PREFIX.len()
|
||||
} else { 0 };
|
||||
let Some(rel_next) = scratchpad[search_from..].find(DELIM_PREFIX) else { break };
|
||||
cursor = search_from + rel_next;
|
||||
truncated = true;
|
||||
}
|
||||
if cursor > 0 {
|
||||
scratchpad.drain(..cursor);
|
||||
}
|
||||
truncated
|
||||
}
|
||||
|
||||
/// Phase 21 — map-reduce over shards with a running scratchpad. See
|
||||
/// module docs.
|
||||
pub async fn generate_tree_split<G: TextGenerator>(
|
||||
generator: &G,
|
||||
shards: &[String],
|
||||
map_prompt: &MapPromptFn<'_>,
|
||||
reduce_prompt: &ReducePromptFn<'_>,
|
||||
opts: &TreeSplitOpts,
|
||||
) -> Result<TreeSplitResult, String> {
|
||||
let mut scratchpad = String::new();
|
||||
let safety = opts.safety_margin.unwrap_or(DEFAULT_SAFETY_MARGIN);
|
||||
let map_max = opts.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS as u32);
|
||||
let reduce_max = opts.reduce_max_tokens.unwrap_or(1_500);
|
||||
let mut truncations = 0usize;
|
||||
let mut total_continuations = 0usize;
|
||||
|
||||
for (i, shard) in shards.iter().enumerate() {
|
||||
let shard_prompt = map_prompt(shard, &scratchpad);
|
||||
// Loud-fail on per-shard overflow — caller sharded too
|
||||
// coarsely. Silent truncation is exactly the mode J rejected.
|
||||
let budget = BudgetOpts {
|
||||
system: opts.system.as_deref(),
|
||||
max_tokens: Some(map_max as usize),
|
||||
safety_margin: Some(safety),
|
||||
bypass: false,
|
||||
};
|
||||
let check = assert_context_budget(&opts.model, &shard_prompt, budget)
|
||||
.map_err(|(c, over)| overflow_message(&opts.model, &c, over, safety))?;
|
||||
let _ = check;
|
||||
|
||||
let mut cont_opts = ContinuableOpts::new(&opts.model);
|
||||
cont_opts.max_tokens = Some(map_max);
|
||||
cont_opts.temperature = opts.temperature;
|
||||
cont_opts.system = opts.system.clone();
|
||||
cont_opts.shape = ResponseShape::Text;
|
||||
cont_opts.think = opts.think;
|
||||
|
||||
let outcome = generate_continuable(generator, &shard_prompt, &cont_opts).await?;
|
||||
total_continuations += outcome.continuations;
|
||||
|
||||
// Append this shard's digest and, if needed, drop oldest.
|
||||
scratchpad.push_str(&format!(
|
||||
"\n— shard {}/{} digest —\n{}",
|
||||
i + 1, shards.len(), outcome.text.trim(),
|
||||
));
|
||||
if truncate_scratchpad(&mut scratchpad, opts.scratchpad_budget) {
|
||||
truncations += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Reduce pass. Budget check first — if the scratchpad is still too
|
||||
// big for the reduce prompt we fail loud with numbers.
|
||||
let reduce_p = reduce_prompt(&scratchpad);
|
||||
let budget = BudgetOpts {
|
||||
system: opts.system.as_deref(),
|
||||
max_tokens: Some(reduce_max as usize),
|
||||
safety_margin: Some(safety),
|
||||
bypass: false,
|
||||
};
|
||||
assert_context_budget(&opts.model, &reduce_p, budget)
|
||||
.map_err(|(c, over)| overflow_message(&opts.model, &c, over, safety))?;
|
||||
|
||||
let mut cont_opts = ContinuableOpts::new(&opts.model);
|
||||
cont_opts.max_tokens = Some(reduce_max);
|
||||
cont_opts.temperature = opts.temperature;
|
||||
cont_opts.system = opts.system.clone();
|
||||
cont_opts.shape = ResponseShape::Text;
|
||||
cont_opts.think = opts.think;
|
||||
|
||||
let outcome = generate_continuable(generator, &reduce_p, &cont_opts).await?;
|
||||
total_continuations += outcome.continuations;
|
||||
|
||||
Ok(TreeSplitResult {
|
||||
response: outcome.text,
|
||||
scratchpad,
|
||||
shards_processed: shards.len(),
|
||||
scratchpad_truncations: truncations,
|
||||
total_continuations,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::continuation::ScriptedGenerator;
|
||||
|
||||
fn simple_map(shard: &str, scratchpad: &str) -> String {
|
||||
format!("SCRATCHPAD:\n{scratchpad}\n---\nSHARD:\n{shard}\n---\nDIGEST:")
|
||||
}
|
||||
|
||||
fn simple_reduce(scratchpad: &str) -> String {
|
||||
format!("SCRATCHPAD:\n{scratchpad}\n---\nFINAL:")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tree_split_runs_map_then_reduce() {
|
||||
// 3 shards → 3 map calls → 1 reduce call = 4 responses scripted.
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("digest-1".to_string()),
|
||||
Ok("digest-2".to_string()),
|
||||
Ok("digest-3".to_string()),
|
||||
Ok("FINAL ANSWER".to_string()),
|
||||
]);
|
||||
let shards: Vec<String> = ["a", "b", "c"].iter().map(|s| s.to_string()).collect();
|
||||
let opts = TreeSplitOpts::new("qwen3:latest");
|
||||
let map_fn: &MapPromptFn = &simple_map;
|
||||
let reduce_fn: &ReducePromptFn = &simple_reduce;
|
||||
let result = generate_tree_split(&generator, &shards, map_fn, reduce_fn, &opts)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.shards_processed, 3);
|
||||
assert_eq!(result.response, "FINAL ANSWER");
|
||||
assert_eq!(generator.call_count(), 4);
|
||||
// Scratchpad must carry all three digests in order.
|
||||
assert!(result.scratchpad.contains("digest-1"));
|
||||
assert!(result.scratchpad.contains("digest-2"));
|
||||
assert!(result.scratchpad.contains("digest-3"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tree_split_reduce_prompt_sees_full_scratchpad() {
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("summary-A".to_string()),
|
||||
Ok("summary-B".to_string()),
|
||||
Ok("REDUCED".to_string()),
|
||||
]);
|
||||
let shards = vec!["input-one".to_string(), "input-two".to_string()];
|
||||
let opts = TreeSplitOpts::new("qwen3:latest");
|
||||
let _ = generate_tree_split(&generator, &shards, &simple_map, &simple_reduce, &opts)
|
||||
.await
|
||||
.unwrap();
|
||||
// Third call = reduce. Its prompt must include both digests.
|
||||
let calls = generator.calls();
|
||||
let reduce_prompt = &calls[2].prompt;
|
||||
assert!(reduce_prompt.contains("summary-A"),
|
||||
"reduce prompt must see first shard digest");
|
||||
assert!(reduce_prompt.contains("summary-B"),
|
||||
"reduce prompt must see second shard digest");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tree_split_loud_fails_on_shard_overflow() {
|
||||
let generator = ScriptedGenerator::new(vec![Ok("digest".to_string())]);
|
||||
// One gigantic shard — well over qwen3's 40K window even as a
|
||||
// prompt. The budget check must reject before any generate call.
|
||||
let shards = vec!["x".repeat(200_000)];
|
||||
let opts = TreeSplitOpts::new("qwen3:latest");
|
||||
let err = generate_tree_split(&generator, &shards, &simple_map, &simple_reduce, &opts)
|
||||
.await
|
||||
.expect_err("shard-sized overflow must be rejected");
|
||||
assert!(err.contains("overflow"), "error should mention overflow: {err}");
|
||||
assert_eq!(generator.call_count(), 0, "generate must not be called on overflow");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tree_split_truncates_scratchpad_when_over_budget() {
|
||||
// Tight budget so each shard trips truncation. qwen3's 40K
|
||||
// window is fine; the budget we care about is the scratchpad
|
||||
// cap, not the model window.
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("A".repeat(2_000)),
|
||||
Ok("B".repeat(2_000)),
|
||||
Ok("C".repeat(2_000)),
|
||||
Ok("D".repeat(2_000)),
|
||||
Ok("FINAL".to_string()),
|
||||
]);
|
||||
let shards: Vec<String> = (0..4).map(|i| format!("shard{i}")).collect();
|
||||
let mut opts = TreeSplitOpts::new("qwen3:latest");
|
||||
opts.scratchpad_budget = 1_000; // ~4000 chars — one digest barely fits
|
||||
let result = generate_tree_split(&generator, &shards, &simple_map, &simple_reduce, &opts)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.scratchpad_truncations > 0,
|
||||
"tight budget must trigger truncation");
|
||||
// Scratchpad should still fit roughly within the budget
|
||||
// (post-truncation); the estimator uses chars/4 so the bound
|
||||
// is ~budget*4 chars. Give some slack for the delimiter.
|
||||
let scratchpad_tokens = ModelMatrix::estimate_tokens(&result.scratchpad);
|
||||
assert!(scratchpad_tokens <= opts.scratchpad_budget * 2,
|
||||
"scratchpad {} tokens vs budget {}", scratchpad_tokens, opts.scratchpad_budget);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tree_split_reports_continuations_from_map_and_reduce() {
|
||||
// First shard: truncated-then-continued. Reduce: truncated-then-continued.
|
||||
// 1 shard: 2 map calls (initial + continuation), then 2 reduce calls.
|
||||
let generator = ScriptedGenerator::new(vec![
|
||||
Ok("partial".to_string()), // map shape=text, non-empty → complete on first pass
|
||||
Ok("reduce-out".to_string()),
|
||||
]);
|
||||
let shards = vec!["only".to_string()];
|
||||
let opts = TreeSplitOpts::new("qwen3:latest");
|
||||
let result = generate_tree_split(&generator, &shards, &simple_map, &simple_reduce, &opts)
|
||||
.await
|
||||
.unwrap();
|
||||
// Text shape treats non-empty as complete → 0 continuations.
|
||||
assert_eq!(result.total_continuations, 0);
|
||||
assert_eq!(result.shards_processed, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_scratchpad_noop_when_under_budget() {
|
||||
let mut s = "\n— shard 1/1 digest —\nshort".to_string();
|
||||
let truncated = truncate_scratchpad(&mut s, 1_000);
|
||||
assert!(!truncated);
|
||||
assert!(s.contains("short"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_scratchpad_drops_oldest_first() {
|
||||
let mut s = format!(
|
||||
"\n— shard 1/3 digest —\n{}\n— shard 2/3 digest —\n{}\n— shard 3/3 digest —\nshort",
|
||||
"x".repeat(4_000), // ~1000 tokens
|
||||
"y".repeat(4_000), // ~1000 tokens
|
||||
);
|
||||
let truncated = truncate_scratchpad(&mut s, 500); // ~2000 chars
|
||||
assert!(truncated);
|
||||
assert!(!s.contains(&"x".repeat(4_000)),
|
||||
"oldest digest should be dropped");
|
||||
assert!(s.contains("short"),
|
||||
"newest digest should survive");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "catalogd"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
storaged = { path = "../storaged" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
object_store = { workspace = true }
|
||||
arrow = { workspace = true }
|
||||
proto = { path = "../proto" }
|
||||
tonic = { workspace = true }
|
||||
@@ -0,0 +1,104 @@
|
||||
use proto::lakehouse::{
|
||||
catalog_service_server::CatalogService,
|
||||
CreateDatasetRequest, DatasetResponse, GetDatasetByNameRequest,
|
||||
GetDatasetRequest, ListDatasetsRequest, ListDatasetsResponse,
|
||||
ObjectRef as ProtoObjectRef,
|
||||
};
|
||||
use shared::types::{DatasetId, ObjectRef, SchemaFingerprint};
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::registry::Registry;
|
||||
|
||||
pub struct CatalogGrpc {
|
||||
registry: Registry,
|
||||
}
|
||||
|
||||
impl CatalogGrpc {
|
||||
pub fn new(registry: Registry) -> Self {
|
||||
Self { registry }
|
||||
}
|
||||
}
|
||||
|
||||
fn manifest_to_proto(m: &shared::types::DatasetManifest) -> DatasetResponse {
|
||||
DatasetResponse {
|
||||
id: m.id.to_string(),
|
||||
name: m.name.clone(),
|
||||
schema_fingerprint: m.schema_fingerprint.0.clone(),
|
||||
objects: m.objects.iter().map(|o| ProtoObjectRef {
|
||||
bucket: o.bucket.clone(),
|
||||
key: o.key.clone(),
|
||||
size_bytes: o.size_bytes,
|
||||
created_at: o.created_at.to_rfc3339(),
|
||||
}).collect(),
|
||||
created_at: m.created_at.to_rfc3339(),
|
||||
updated_at: m.updated_at.to_rfc3339(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl CatalogService for CatalogGrpc {
|
||||
async fn create_dataset(
|
||||
&self,
|
||||
request: Request<CreateDatasetRequest>,
|
||||
) -> Result<Response<DatasetResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let now = chrono::Utc::now();
|
||||
let objects: Vec<ObjectRef> = req.objects.into_iter().map(|o| ObjectRef {
|
||||
bucket: o.bucket,
|
||||
key: o.key,
|
||||
size_bytes: o.size_bytes,
|
||||
created_at: now,
|
||||
}).collect();
|
||||
|
||||
let manifest = self.registry
|
||||
.register(req.name, SchemaFingerprint(req.schema_fingerprint), objects)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
// Mirror the HTTP side's 409 mapping: schema drift is a
|
||||
// precondition failure, not a server error. Keeps gRPC
|
||||
// and HTTP callers seeing the same diagnostic signal.
|
||||
if e.contains("different schema") {
|
||||
Status::failed_precondition(e)
|
||||
} else {
|
||||
Status::internal(e)
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(Response::new(manifest_to_proto(&manifest)))
|
||||
}
|
||||
|
||||
async fn get_dataset(
|
||||
&self,
|
||||
request: Request<GetDatasetRequest>,
|
||||
) -> Result<Response<DatasetResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let uuid = Uuid::parse_str(&req.id).map_err(|e| Status::invalid_argument(e.to_string()))?;
|
||||
let id = DatasetId(uuid);
|
||||
|
||||
match self.registry.get(&id).await {
|
||||
Some(m) => Ok(Response::new(manifest_to_proto(&m))),
|
||||
None => Err(Status::not_found(format!("dataset not found: {}", req.id))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_dataset_by_name(
|
||||
&self,
|
||||
request: Request<GetDatasetByNameRequest>,
|
||||
) -> Result<Response<DatasetResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
match self.registry.get_by_name(&req.name).await {
|
||||
Some(m) => Ok(Response::new(manifest_to_proto(&m))),
|
||||
None => Err(Status::not_found(format!("dataset not found: {}", req.name))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_datasets(
|
||||
&self,
|
||||
_request: Request<ListDatasetsRequest>,
|
||||
) -> Result<Response<ListDatasetsResponse>, Status> {
|
||||
let datasets = self.registry.list().await;
|
||||
let responses = datasets.iter().map(manifest_to_proto).collect();
|
||||
Ok(Response::new(ListDatasetsResponse { datasets: responses }))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod registry;
|
||||
pub mod service;
|
||||
pub mod grpc;
|
||||
pub mod tombstones;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,475 @@
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use shared::types::{DatasetId, ObjectRef, SchemaFingerprint};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::registry::Registry;
|
||||
|
||||
pub fn router(registry: Registry) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/datasets", post(create_dataset))
|
||||
.route("/datasets", get(list_datasets))
|
||||
.route("/datasets/{id}", get(get_dataset))
|
||||
.route("/datasets/by-name/{name}", get(get_dataset_by_name).delete(delete_dataset_by_name))
|
||||
.route("/datasets/by-name/{name}/metadata", post(update_metadata))
|
||||
.route("/datasets/by-name/{name}/resync", post(resync_dataset))
|
||||
.route("/resync-missing", post(resync_all_missing))
|
||||
.route("/migrate-buckets", post(migrate_buckets))
|
||||
.route("/dedupe", post(dedupe_by_name))
|
||||
// Phase D: AI-safe views
|
||||
.route("/views", post(create_view).get(list_views))
|
||||
.route("/views/{name}", get(get_view).delete(delete_view))
|
||||
// Phase E: soft-delete tombstones
|
||||
.route("/datasets/by-name/{name}/tombstone", post(tombstone_rows).get(list_tombstones))
|
||||
// Phase 17: model profiles
|
||||
.route("/profiles", post(create_profile).get(list_profiles))
|
||||
.route("/profiles/{id}", get(get_profile).delete(delete_profile))
|
||||
// Phase 41: profile-type routes
|
||||
.route("/profiles/retrieval", get(list_retrieval_profiles))
|
||||
.route("/profiles/memory", get(list_memory_profiles))
|
||||
.route("/profiles/observer", get(list_observer_profiles))
|
||||
.with_state(registry)
|
||||
}
|
||||
|
||||
async fn health() -> &'static str {
|
||||
"catalogd ok"
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateDatasetRequest {
|
||||
name: String,
|
||||
schema_fingerprint: String,
|
||||
objects: Vec<ObjectRefRequest>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ObjectRefRequest {
|
||||
bucket: String,
|
||||
key: String,
|
||||
size_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct DatasetResponse {
|
||||
id: String,
|
||||
name: String,
|
||||
schema_fingerprint: String,
|
||||
objects: Vec<ObjectRefResponse>,
|
||||
created_at: String,
|
||||
updated_at: String,
|
||||
// Rich metadata
|
||||
description: String,
|
||||
owner: String,
|
||||
sensitivity: Option<shared::types::Sensitivity>,
|
||||
columns: Vec<shared::types::ColumnMeta>,
|
||||
lineage: Option<shared::types::Lineage>,
|
||||
freshness: Option<shared::types::FreshnessContract>,
|
||||
tags: Vec<String>,
|
||||
row_count: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ObjectRefResponse {
|
||||
bucket: String,
|
||||
key: String,
|
||||
size_bytes: u64,
|
||||
created_at: String,
|
||||
}
|
||||
|
||||
impl From<&shared::types::DatasetManifest> for DatasetResponse {
|
||||
fn from(m: &shared::types::DatasetManifest) -> Self {
|
||||
Self {
|
||||
id: m.id.to_string(),
|
||||
name: m.name.clone(),
|
||||
schema_fingerprint: m.schema_fingerprint.0.clone(),
|
||||
objects: m.objects.iter().map(|o| ObjectRefResponse {
|
||||
bucket: o.bucket.clone(),
|
||||
key: o.key.clone(),
|
||||
size_bytes: o.size_bytes,
|
||||
created_at: o.created_at.to_rfc3339(),
|
||||
}).collect(),
|
||||
created_at: m.created_at.to_rfc3339(),
|
||||
updated_at: m.updated_at.to_rfc3339(),
|
||||
description: m.description.clone(),
|
||||
owner: m.owner.clone(),
|
||||
sensitivity: m.sensitivity.clone(),
|
||||
columns: m.columns.clone(),
|
||||
lineage: m.lineage.clone(),
|
||||
freshness: m.freshness.clone(),
|
||||
tags: m.tags.clone(),
|
||||
row_count: m.row_count,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_dataset(
|
||||
State(registry): State<Registry>,
|
||||
Json(req): Json<CreateDatasetRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let now = chrono::Utc::now();
|
||||
let objects: Vec<ObjectRef> = req.objects.into_iter().map(|o| ObjectRef {
|
||||
bucket: o.bucket,
|
||||
key: o.key,
|
||||
size_bytes: o.size_bytes,
|
||||
created_at: now,
|
||||
}).collect();
|
||||
|
||||
match registry.register(req.name, SchemaFingerprint(req.schema_fingerprint), objects).await {
|
||||
Ok(manifest) => {
|
||||
let resp = DatasetResponse::from(&manifest);
|
||||
Ok((StatusCode::CREATED, Json(resp)))
|
||||
}
|
||||
Err(e) => {
|
||||
let status = if e.contains("different schema") {
|
||||
StatusCode::CONFLICT
|
||||
} else {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
};
|
||||
Err((status, e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_datasets(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
let datasets = registry.list().await;
|
||||
let resp: Vec<DatasetResponse> = datasets.iter().map(DatasetResponse::from).collect();
|
||||
Json(resp)
|
||||
}
|
||||
|
||||
async fn get_dataset(
|
||||
State(registry): State<Registry>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
let uuid = Uuid::parse_str(&id).map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
|
||||
let dataset_id = DatasetId(uuid);
|
||||
match registry.get(&dataset_id).await {
|
||||
Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {id}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_dataset_by_name(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.get_by_name(&name).await {
|
||||
Some(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("dataset not found: {name}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a dataset manifest by name. Metadata only — parquet files,
|
||||
/// vector indexes, and tombstones are NOT cascade-deleted. See
|
||||
/// `Registry::delete_dataset` for the full scope.
|
||||
async fn delete_dataset_by_name(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.delete_dataset(&name).await {
|
||||
Ok(removed) => {
|
||||
let body = serde_json::json!({ "name": name, "manifests_removed": removed });
|
||||
Ok(Json(body))
|
||||
}
|
||||
Err(e) if e.starts_with("dataset not found") => {
|
||||
Err((StatusCode::NOT_FOUND, e))
|
||||
}
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_metadata(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
Json(updates): Json<crate::registry::MetadataUpdate>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.update_metadata(&name, updates).await {
|
||||
Ok(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-read parquet footers for a single dataset and repopulate row_count
|
||||
/// and columns from reality. Useful for repairing manifests whose metadata
|
||||
/// was lost or never backfilled.
|
||||
async fn resync_dataset(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.resync_from_parquet(&name).await {
|
||||
Ok(manifest) => Ok(Json(DatasetResponse::from(&manifest))),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ResyncAllResponse {
|
||||
succeeded: Vec<ResyncOk>,
|
||||
failed: Vec<ResyncErr>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ResyncOk {
|
||||
name: String,
|
||||
row_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ResyncErr {
|
||||
name: String,
|
||||
error: String,
|
||||
}
|
||||
|
||||
/// Resync every dataset that currently has null row_count or empty columns.
|
||||
async fn resync_all_missing(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
let (ok, err) = registry.resync_missing().await;
|
||||
Json(ResyncAllResponse {
|
||||
succeeded: ok.into_iter().map(|(name, row_count)| ResyncOk { name, row_count }).collect(),
|
||||
failed: err.into_iter().map(|(name, error)| ResyncErr { name, error }).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Federation layer 2 one-shot: normalize every ObjectRef.bucket field
|
||||
/// to the canonical "primary" value. Idempotent — re-running once
|
||||
/// everything is canonical is a safe no-op.
|
||||
async fn migrate_buckets(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
match registry.migrate_buckets_to_primary().await {
|
||||
Ok(report) => Ok(Json(report)),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Collapse duplicate manifests by `name` — see `Registry::dedupe_by_name`.
|
||||
/// Safe to run repeatedly; single-manifest datasets are untouched.
|
||||
async fn dedupe_by_name(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
Json(registry.dedupe_by_name().await)
|
||||
}
|
||||
|
||||
// --- Phase D: AI-safe views ---
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateViewRequest {
|
||||
name: String,
|
||||
base_dataset: String,
|
||||
columns: Vec<String>,
|
||||
#[serde(default)]
|
||||
row_filter: Option<String>,
|
||||
#[serde(default)]
|
||||
column_redactions: std::collections::HashMap<String, shared::types::Redaction>,
|
||||
#[serde(default)]
|
||||
description: String,
|
||||
#[serde(default)]
|
||||
created_by: String,
|
||||
}
|
||||
|
||||
async fn create_view(
|
||||
State(registry): State<Registry>,
|
||||
Json(req): Json<CreateViewRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let view = shared::types::AiView {
|
||||
name: req.name,
|
||||
base_dataset: req.base_dataset,
|
||||
columns: req.columns,
|
||||
row_filter: req.row_filter,
|
||||
column_redactions: req.column_redactions,
|
||||
created_at: chrono::Utc::now(),
|
||||
created_by: req.created_by,
|
||||
description: req.description,
|
||||
};
|
||||
match registry.put_view(view).await {
|
||||
Ok(v) => Ok((StatusCode::CREATED, Json(v))),
|
||||
Err(e) => Err((StatusCode::BAD_REQUEST, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_views(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
Json(registry.list_views().await)
|
||||
}
|
||||
|
||||
async fn get_view(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.get_view(&name).await {
|
||||
Some(v) => Ok(Json(v)),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("view not found: {name}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_view(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.delete_view(&name).await {
|
||||
Ok(()) => Ok(StatusCode::NO_CONTENT),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Phase E: soft-delete tombstones ---
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct TombstoneRequest {
|
||||
row_key_column: String,
|
||||
row_key_values: Vec<String>,
|
||||
#[serde(default)]
|
||||
actor: String,
|
||||
#[serde(default)]
|
||||
reason: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct TombstoneResponse {
|
||||
dataset: String,
|
||||
row_key_column: String,
|
||||
rows_tombstoned: usize,
|
||||
failures: Vec<String>,
|
||||
}
|
||||
|
||||
async fn tombstone_rows(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
Json(req): Json<TombstoneRequest>,
|
||||
) -> impl IntoResponse {
|
||||
if req.row_key_values.is_empty() {
|
||||
return Err((StatusCode::BAD_REQUEST, "row_key_values is empty".to_string()));
|
||||
}
|
||||
|
||||
let mut ok = 0;
|
||||
let mut failures = Vec::new();
|
||||
for value in &req.row_key_values {
|
||||
match registry
|
||||
.add_tombstone(&name, &req.row_key_column, value, &req.actor, &req.reason)
|
||||
.await
|
||||
{
|
||||
Ok(_) => ok += 1,
|
||||
Err(e) => failures.push(format!("{value}: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
let status = if ok > 0 && failures.is_empty() {
|
||||
StatusCode::CREATED
|
||||
} else if ok > 0 {
|
||||
StatusCode::MULTI_STATUS
|
||||
} else {
|
||||
StatusCode::BAD_REQUEST
|
||||
};
|
||||
|
||||
Ok((status, Json(TombstoneResponse {
|
||||
dataset: name,
|
||||
row_key_column: req.row_key_column,
|
||||
rows_tombstoned: ok,
|
||||
failures,
|
||||
})))
|
||||
}
|
||||
|
||||
async fn list_tombstones(
|
||||
State(registry): State<Registry>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.list_tombstones(&name).await {
|
||||
Ok(ts) => Ok(Json(ts)),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Phase 17: Model profiles ---
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateProfileRequest {
|
||||
id: String,
|
||||
ollama_name: String,
|
||||
#[serde(default)]
|
||||
description: String,
|
||||
bound_datasets: Vec<String>,
|
||||
#[serde(default)]
|
||||
hnsw_config: shared::types::ProfileHnswConfig,
|
||||
#[serde(default = "default_embed_model_req")]
|
||||
embed_model: String,
|
||||
#[serde(default)]
|
||||
created_by: String,
|
||||
/// Federation: optional per-profile bucket (`profile:{id}` by convention).
|
||||
/// Omitting keeps artifacts in primary.
|
||||
#[serde(default)]
|
||||
bucket: Option<String>,
|
||||
/// ADR-019 hybrid: which vector backend to route this profile's
|
||||
/// indexes to. Defaults to Parquet+HNSW.
|
||||
#[serde(default)]
|
||||
vector_backend: shared::types::VectorBackend,
|
||||
/// Phase 41: Profile type for routing + activation behavior.
|
||||
#[serde(default)]
|
||||
profile_type: shared::types::ProfileType,
|
||||
}
|
||||
|
||||
fn default_embed_model_req() -> String { "nomic-embed-text".to_string() }
|
||||
|
||||
async fn create_profile(
|
||||
State(registry): State<Registry>,
|
||||
Json(req): Json<CreateProfileRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let profile = shared::types::ModelProfile {
|
||||
id: req.id,
|
||||
ollama_name: req.ollama_name,
|
||||
description: req.description,
|
||||
bound_datasets: req.bound_datasets,
|
||||
hnsw_config: req.hnsw_config,
|
||||
embed_model: req.embed_model,
|
||||
created_at: chrono::Utc::now(),
|
||||
created_by: req.created_by,
|
||||
bucket: req.bucket,
|
||||
vector_backend: req.vector_backend,
|
||||
profile_type: req.profile_type,
|
||||
};
|
||||
match registry.put_profile(profile).await {
|
||||
Ok(p) => Ok((StatusCode::CREATED, Json(p))),
|
||||
Err(e) => Err((StatusCode::BAD_REQUEST, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_profiles(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
Json(registry.list_profiles().await)
|
||||
}
|
||||
|
||||
async fn list_retrieval_profiles(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
let all = registry.list_profiles().await;
|
||||
let retrieval: Vec<_> = all.into_iter().filter(|p| p.profile_type == shared::types::ProfileType::Retrieval).collect();
|
||||
Json(retrieval)
|
||||
}
|
||||
|
||||
async fn list_memory_profiles(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
let all = registry.list_profiles().await;
|
||||
let memory: Vec<_> = all.into_iter().filter(|p| p.profile_type == shared::types::ProfileType::Memory).collect();
|
||||
Json(memory)
|
||||
}
|
||||
|
||||
async fn list_observer_profiles(State(registry): State<Registry>) -> impl IntoResponse {
|
||||
let all = registry.list_profiles().await;
|
||||
let observer: Vec<_> = all.into_iter().filter(|p| p.profile_type == shared::types::ProfileType::Observer).collect();
|
||||
Json(observer)
|
||||
}
|
||||
|
||||
async fn get_profile(
|
||||
State(registry): State<Registry>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.get_profile(&id).await {
|
||||
Some(p) => Ok(Json(p)),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("profile not found: {id}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_profile(
|
||||
State(registry): State<Registry>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match registry.delete_profile(&id).await {
|
||||
Ok(()) => Ok(StatusCode::NO_CONTENT),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
//! Soft-delete tombstone storage (Phase E).
|
||||
//!
|
||||
//! One append-log per dataset at `_catalog/tombstones/{dataset}/batch_*.jsonl`.
|
||||
//! Uses the shared `storaged::append_log::AppendLog` pattern so appends
|
||||
//! are write-once (never rewrites existing files) and can be compacted.
|
||||
//!
|
||||
//! The store exposes a per-dataset cache of active tombstones for the
|
||||
//! hot path (queryd filter construction) — that's why events are pulled
|
||||
//! into an in-memory map on every call rather than scanning object
|
||||
//! storage repeatedly.
|
||||
|
||||
use object_store::ObjectStore;
|
||||
use shared::types::Tombstone;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use storaged::append_log::{AppendLog, CompactStats};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
const TOMBSTONE_PREFIX: &str = "_catalog/tombstones";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TombstoneStore {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
logs: Arc<RwLock<HashMap<String, Arc<AppendLog>>>>,
|
||||
}
|
||||
|
||||
impl TombstoneStore {
|
||||
pub fn new(store: Arc<dyn ObjectStore>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
logs: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
fn prefix_for(dataset: &str) -> String {
|
||||
// Sanitize dataset name for filesystem safety.
|
||||
let safe: String = dataset
|
||||
.chars()
|
||||
.map(|c| if c.is_ascii_alphanumeric() || c == '_' || c == '-' { c } else { '_' })
|
||||
.collect();
|
||||
format!("{TOMBSTONE_PREFIX}/{}", safe)
|
||||
}
|
||||
|
||||
async fn log_for(&self, dataset: &str) -> Arc<AppendLog> {
|
||||
if let Some(log) = self.logs.read().await.get(dataset) {
|
||||
return log.clone();
|
||||
}
|
||||
let mut guard = self.logs.write().await;
|
||||
if let Some(log) = guard.get(dataset) {
|
||||
return log.clone();
|
||||
}
|
||||
// Threshold of 1 — every tombstone is high-value. Compliance/audit
|
||||
// doesn't tolerate "lost on restart"; we trade a small file count
|
||||
// for guaranteed durability. Compaction merges later if volume grows.
|
||||
let log = Arc::new(
|
||||
AppendLog::new(self.store.clone(), Self::prefix_for(dataset))
|
||||
.with_flush_threshold(1),
|
||||
);
|
||||
guard.insert(dataset.to_string(), log.clone());
|
||||
log
|
||||
}
|
||||
|
||||
/// Append one tombstone. Validates that the `row_key_column` matches
|
||||
/// the column already used for this dataset (all tombstones for a
|
||||
/// dataset share one key column so the query filter is well-defined).
|
||||
/// Forces a flush so the tombstone is durable before this call returns.
|
||||
pub async fn append(&self, ts: &Tombstone) -> Result<(), String> {
|
||||
let existing = self.list(&ts.dataset).await?;
|
||||
if let Some(prior) = existing.first() {
|
||||
if prior.row_key_column != ts.row_key_column {
|
||||
return Err(format!(
|
||||
"dataset '{}' already uses '{}' as tombstone key; cannot mix with '{}'",
|
||||
ts.dataset, prior.row_key_column, ts.row_key_column,
|
||||
));
|
||||
}
|
||||
}
|
||||
let line = serde_json::to_vec(ts).map_err(|e| e.to_string())?;
|
||||
let log = self.log_for(&ts.dataset).await;
|
||||
log.append(line).await?;
|
||||
// Belt-and-suspenders: explicit flush in case the threshold is
|
||||
// ever raised. Tombstones must be durable on return.
|
||||
log.flush().await
|
||||
}
|
||||
|
||||
/// All tombstones for a dataset (chronological).
|
||||
pub async fn list(&self, dataset: &str) -> Result<Vec<Tombstone>, String> {
|
||||
let log = self.log_for(dataset).await;
|
||||
let lines = log.read_all().await?;
|
||||
let mut out = Vec::with_capacity(lines.len());
|
||||
for line in lines {
|
||||
match serde_json::from_slice::<Tombstone>(&line) {
|
||||
Ok(t) => out.push(t),
|
||||
Err(e) => tracing::warn!("tombstones/{}: skip malformed entry: {e}", dataset),
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Per-dataset grouped view used by queryd — returns a map of
|
||||
/// `{dataset -> (row_key_column, set_of_values)}` for every dataset
|
||||
/// that has any tombstones.
|
||||
pub async fn all_grouped(
|
||||
&self,
|
||||
datasets: &[String],
|
||||
) -> Result<HashMap<String, (String, Vec<String>)>, String> {
|
||||
let mut grouped = HashMap::new();
|
||||
for dataset in datasets {
|
||||
let ts = match self.list(dataset).await {
|
||||
Ok(ts) => ts,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ts.is_empty() { continue; }
|
||||
let col = ts[0].row_key_column.clone();
|
||||
let values: Vec<String> = ts.iter().map(|t| t.row_key_value.clone()).collect();
|
||||
grouped.insert(dataset.clone(), (col, values));
|
||||
}
|
||||
Ok(grouped)
|
||||
}
|
||||
|
||||
pub async fn compact(&self, dataset: &str) -> Result<CompactStats, String> {
|
||||
let log = self.log_for(dataset).await;
|
||||
log.compact().await
|
||||
}
|
||||
|
||||
/// Remove every tombstone for a dataset. Called after a successful
|
||||
/// parquet compaction has physically deleted those rows — the
|
||||
/// tombstones have done their job and the journal can be cleared.
|
||||
///
|
||||
/// Implementation: drop the per-dataset AppendLog from the cache and
|
||||
/// delete all its batch files. Next write starts fresh.
|
||||
pub async fn clear(&self, dataset: &str) -> Result<usize, String> {
|
||||
let prefix = format!("{}/", Self::prefix_for(dataset));
|
||||
let keys = storaged::ops::list(&self.store, Some(&prefix)).await?;
|
||||
let matching: Vec<String> = keys
|
||||
.into_iter()
|
||||
.filter(|k| {
|
||||
let basename = k.rsplit('/').next().unwrap_or(k);
|
||||
basename.starts_with("batch_") && basename.ends_with(".jsonl")
|
||||
})
|
||||
.collect();
|
||||
let count = matching.len();
|
||||
for key in &matching {
|
||||
let _ = storaged::ops::delete(&self.store, key).await;
|
||||
}
|
||||
self.logs.write().await.remove(dataset);
|
||||
if count > 0 {
|
||||
tracing::info!("cleared {count} tombstone batch files for '{}'", dataset);
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "gateway"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
storaged = { path = "../storaged" }
|
||||
catalogd = { path = "../catalogd" }
|
||||
queryd = { path = "../queryd" }
|
||||
aibridge = { path = "../aibridge" }
|
||||
ingestd = { path = "../ingestd" }
|
||||
vectord = { path = "../vectord" }
|
||||
journald = { path = "../journald" }
|
||||
truth = { path = "../truth" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
tower-http = { workspace = true }
|
||||
object_store = { workspace = true }
|
||||
proto = { path = "../proto" }
|
||||
tonic = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true }
|
||||
opentelemetry-stdout = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
arrow = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
@@ -0,0 +1,160 @@
|
||||
/// Access control: field-level sensitivity enforcement, column masking, query audit.
|
||||
/// Evaluates policies at query time — not at storage level.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use shared::types::Sensitivity;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// An agent's role determines what sensitivity levels they can see.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AgentRole {
|
||||
pub agent_name: String,
|
||||
pub role: String, // "recruiter", "admin", "analyst", "agent"
|
||||
pub allowed_sensitivity: Vec<Sensitivity>, // what they can see unmasked
|
||||
}
|
||||
|
||||
/// A query audit entry.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QueryAudit {
|
||||
pub id: String,
|
||||
pub agent: String,
|
||||
pub sql: String,
|
||||
pub datasets_accessed: Vec<String>,
|
||||
pub pii_fields_accessed: Vec<String>,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub row_count: usize,
|
||||
pub allowed: bool,
|
||||
pub masked_fields: Vec<String>,
|
||||
}
|
||||
|
||||
/// Access control manager.
|
||||
#[derive(Clone)]
|
||||
pub struct AccessControl {
|
||||
roles: Arc<RwLock<HashMap<String, AgentRole>>>,
|
||||
audit_log: Arc<RwLock<Vec<QueryAudit>>>,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl AccessControl {
|
||||
pub fn new(enabled: bool) -> Self {
|
||||
let ac = Self {
|
||||
roles: Arc::new(RwLock::new(HashMap::new())),
|
||||
audit_log: Arc::new(RwLock::new(Vec::new())),
|
||||
enabled,
|
||||
};
|
||||
ac
|
||||
}
|
||||
|
||||
/// Register default roles.
|
||||
pub async fn register_defaults(&self) {
|
||||
let defaults = vec![
|
||||
AgentRole {
|
||||
agent_name: "admin".into(),
|
||||
role: "admin".into(),
|
||||
allowed_sensitivity: vec![
|
||||
Sensitivity::Public, Sensitivity::Internal,
|
||||
Sensitivity::Pii, Sensitivity::Phi, Sensitivity::Financial,
|
||||
],
|
||||
},
|
||||
AgentRole {
|
||||
agent_name: "recruiter".into(),
|
||||
role: "recruiter".into(),
|
||||
allowed_sensitivity: vec![
|
||||
Sensitivity::Public, Sensitivity::Internal, Sensitivity::Pii,
|
||||
],
|
||||
},
|
||||
AgentRole {
|
||||
agent_name: "analyst".into(),
|
||||
role: "analyst".into(),
|
||||
allowed_sensitivity: vec![
|
||||
Sensitivity::Public, Sensitivity::Internal, Sensitivity::Financial,
|
||||
],
|
||||
},
|
||||
AgentRole {
|
||||
agent_name: "agent".into(),
|
||||
role: "agent".into(),
|
||||
allowed_sensitivity: vec![
|
||||
Sensitivity::Public, Sensitivity::Internal,
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
let mut roles = self.roles.write().await;
|
||||
for role in defaults {
|
||||
roles.insert(role.agent_name.clone(), role);
|
||||
}
|
||||
}
|
||||
|
||||
/// Register or update an agent role.
|
||||
pub async fn set_role(&self, role: AgentRole) {
|
||||
self.roles.write().await.insert(role.agent_name.clone(), role);
|
||||
}
|
||||
|
||||
/// Get an agent's role. Called by `GET /access/roles/{agent}`.
|
||||
pub async fn get_role(&self, agent: &str) -> Option<AgentRole> {
|
||||
self.roles.read().await.get(agent).cloned()
|
||||
}
|
||||
|
||||
/// List all roles.
|
||||
pub async fn list_roles(&self) -> Vec<AgentRole> {
|
||||
self.roles.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// Check if an agent can see a field with given sensitivity.
|
||||
pub async fn can_access(&self, agent: &str, sensitivity: &Sensitivity) -> bool {
|
||||
if !self.enabled { return true; }
|
||||
match self.roles.read().await.get(agent) {
|
||||
Some(role) => role.allowed_sensitivity.contains(sensitivity),
|
||||
None => false, // unknown agent = no access
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine which fields should be masked for an agent.
|
||||
#[allow(dead_code)]
|
||||
pub async fn masked_fields(
|
||||
&self,
|
||||
agent: &str,
|
||||
columns: &[shared::types::ColumnMeta],
|
||||
) -> Vec<String> {
|
||||
if !self.enabled { return vec![]; }
|
||||
|
||||
let role = match self.roles.read().await.get(agent) {
|
||||
Some(r) => r.clone(),
|
||||
None => return columns.iter().filter(|c| c.sensitivity.is_some()).map(|c| c.name.clone()).collect(),
|
||||
};
|
||||
|
||||
columns.iter()
|
||||
.filter(|col| {
|
||||
if let Some(ref sens) = col.sensitivity {
|
||||
!role.allowed_sensitivity.contains(sens)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
.map(|col| col.name.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Log a query for audit.
|
||||
#[allow(dead_code)]
|
||||
pub async fn log_query(&self, audit: QueryAudit) {
|
||||
self.audit_log.write().await.push(audit);
|
||||
}
|
||||
|
||||
/// Get recent audit entries.
|
||||
pub async fn recent_audit(&self, limit: usize) -> Vec<QueryAudit> {
|
||||
let log = self.audit_log.read().await;
|
||||
let start = log.len().saturating_sub(limit);
|
||||
log[start..].iter().rev().cloned().collect()
|
||||
}
|
||||
|
||||
/// Reports whether access-control enforcement is active.
|
||||
/// Called by `GET /access/enabled` — ops tooling / dashboards poll
|
||||
/// this to confirm the auth posture of the running gateway.
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::access::{AccessControl, AgentRole};
|
||||
|
||||
pub fn router(ac: AccessControl) -> Router {
|
||||
Router::new()
|
||||
.route("/roles", get(list_roles))
|
||||
.route("/roles", post(set_role))
|
||||
// Scrum iter 11 / P13-001 finding: get_role was #[allow(dead_code)]
|
||||
// because nothing called it — dead until exposed. Route activates it.
|
||||
// Returns 404 when the agent isn't registered so clients can
|
||||
// distinguish "missing role" from "access denied."
|
||||
.route("/roles/{agent}", get(get_role))
|
||||
.route("/enabled", get(enabled_status))
|
||||
.route("/audit", get(query_audit))
|
||||
.route("/check", post(check_access))
|
||||
.with_state(ac)
|
||||
}
|
||||
|
||||
async fn list_roles(State(ac): State<AccessControl>) -> impl IntoResponse {
|
||||
Json(ac.list_roles().await)
|
||||
}
|
||||
|
||||
async fn set_role(
|
||||
State(ac): State<AccessControl>,
|
||||
Json(role): Json<AgentRole>,
|
||||
) -> impl IntoResponse {
|
||||
let name = role.agent_name.clone();
|
||||
ac.set_role(role).await;
|
||||
(StatusCode::OK, format!("role set: {name}"))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AuditQuery {
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
async fn query_audit(
|
||||
State(ac): State<AccessControl>,
|
||||
Query(q): Query<AuditQuery>,
|
||||
) -> impl IntoResponse {
|
||||
Json(ac.recent_audit(q.limit.unwrap_or(50)).await)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CheckRequest {
|
||||
agent: String,
|
||||
sensitivity: shared::types::Sensitivity,
|
||||
}
|
||||
|
||||
async fn check_access(
|
||||
State(ac): State<AccessControl>,
|
||||
Json(req): Json<CheckRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let allowed = ac.can_access(&req.agent, &req.sensitivity).await;
|
||||
Json(serde_json::json!({
|
||||
"agent": req.agent,
|
||||
"sensitivity": req.sensitivity,
|
||||
"allowed": allowed,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_role(
|
||||
State(ac): State<AccessControl>,
|
||||
Path(agent): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match ac.get_role(&agent).await {
|
||||
Some(role) => Ok(Json(role)),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("no role registered for agent '{agent}'"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn enabled_status(State(ac): State<AccessControl>) -> impl IntoResponse {
|
||||
Json(serde_json::json!({ "enabled": ac.is_enabled() }))
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
use axum::{
|
||||
extract::Request,
|
||||
http::StatusCode,
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
|
||||
// API key auth middleware. Checks X-API-Key header against configured key.
|
||||
// Fixed P5-001 (2026-04-23): previously #[allow(dead_code)] — the function
|
||||
// existed but was never layered onto the router, so [auth] enabled=true
|
||||
// silently enforced nothing. Now wired via from_fn_with_state in main.rs.
|
||||
pub async fn api_key_auth(
|
||||
axum::extract::State(expected): axum::extract::State<ApiKey>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Result<Response, StatusCode> {
|
||||
// /health stays public (LB/systemd probes). Every other route is gated.
|
||||
if request.uri().path() == "/health" {
|
||||
return Ok(next.run(request).await);
|
||||
}
|
||||
|
||||
let provided = request
|
||||
.headers()
|
||||
.get("x-api-key")
|
||||
.and_then(|v| v.to_str().ok());
|
||||
|
||||
// Constant-time-ish eq on the raw bytes; good enough for a shared-secret
|
||||
// X-API-Key. Timing-attack resistance here matters less than the
|
||||
// equivalent HMAC check would; adopt subtle crate if key-space grows.
|
||||
match provided {
|
||||
Some(key) if eq_ct(key.as_bytes(), expected.0.as_bytes()) => {
|
||||
Ok(next.run(request).await)
|
||||
}
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
path = %request.uri().path(),
|
||||
"unauthorized request: missing or invalid API key",
|
||||
);
|
||||
Err(StatusCode::UNAUTHORIZED)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn eq_ct(a: &[u8], b: &[u8]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
let mut diff: u8 = 0;
|
||||
for (x, y) in a.iter().zip(b.iter()) {
|
||||
diff |= x ^ y;
|
||||
}
|
||||
diff == 0
|
||||
}
|
||||
|
||||
/// Wrapper type for the API key, stored in request extensions.
|
||||
#[derive(Clone)]
|
||||
pub struct ApiKey(pub String);
|
||||
@@ -0,0 +1,388 @@
|
||||
//! KB context loader — reads recent signal from `data/_kb/*.jsonl` for
|
||||
//! a given sig_hash + task_class and returns a compact summary.
|
||||
//!
|
||||
//! This is the "pipe to the overviewer" from the 2026-04-23 session:
|
||||
//! the overseer tier (T3, gpt-oss:120b) consumes this context before
|
||||
//! generating a correction, so its suggestions are informed by
|
||||
//! historical cost / latency / outcome / prior-correction patterns
|
||||
//! across ALL profiles that have run this task class — not just the
|
||||
//! single current loop.
|
||||
//!
|
||||
//! Hot-swap profiles read the SAME pool. When a profile activates and
|
||||
//! starts iterating, its KB context is the shared surface — one
|
||||
//! profile's learning becomes every profile's starting point.
|
||||
//!
|
||||
//! Best-effort throughout: missing files, corrupt rows, empty
|
||||
//! directories all produce an empty KbContext. The overseer works
|
||||
//! fine with no history; we just can't seed it then.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::path::Path;
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
|
||||
/// Compact summary returned to the overseer. Bounded size — recent
|
||||
/// outcomes + corrections plus rolled-up rates. Goal is to fit in a
|
||||
/// prompt without eating the overseer's context budget.
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct KbContext {
|
||||
pub sig_hash: String,
|
||||
pub task_class: String,
|
||||
pub recent_outcomes: Vec<OutcomeSummary>,
|
||||
pub recent_corrections: Vec<CorrectionSummary>,
|
||||
pub success_rate: Option<f64>,
|
||||
pub avg_turns: Option<f64>,
|
||||
pub avg_latency_ms: Option<u64>,
|
||||
pub total_observed: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OutcomeSummary {
|
||||
pub created_at: String,
|
||||
pub ok: bool,
|
||||
pub polarity: String,
|
||||
pub turns: u32,
|
||||
pub latency_ms: u64,
|
||||
pub total_tokens: u64,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct CorrectionSummary {
|
||||
pub created_at: String,
|
||||
pub reason: String,
|
||||
pub correction_preview: String, // first 300 chars
|
||||
pub applied_at_turn: u32,
|
||||
}
|
||||
|
||||
const OUTCOMES_PATH: &str = "data/_kb/outcomes.jsonl";
|
||||
const CORRECTIONS_PATH: &str = "data/_kb/overseer_corrections.jsonl";
|
||||
const RECENT_OUTCOME_LIMIT: usize = 5;
|
||||
const RECENT_CORRECTION_LIMIT: usize = 3;
|
||||
const AGGREGATE_WINDOW: usize = 50;
|
||||
|
||||
impl KbContext {
|
||||
/// Build context from the default KB paths.
|
||||
pub async fn load_for(sig_hash: &str, task_class: &str) -> Self {
|
||||
Self::load_from(
|
||||
sig_hash, task_class,
|
||||
Path::new(OUTCOMES_PATH), Path::new(CORRECTIONS_PATH),
|
||||
).await
|
||||
}
|
||||
|
||||
/// Path-taking variant — tests inject tmp files without touching
|
||||
/// the real KB directory (same pattern as append_outcomes_row_at).
|
||||
pub async fn load_from(
|
||||
sig_hash: &str,
|
||||
task_class: &str,
|
||||
outcomes_path: &Path,
|
||||
corrections_path: &Path,
|
||||
) -> Self {
|
||||
let mut ctx = KbContext {
|
||||
sig_hash: sig_hash.to_string(),
|
||||
task_class: task_class.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Scan outcomes — matches on sig_hash primary, task_class
|
||||
// secondary (so different geos for the same task_class still
|
||||
// contribute to aggregate rates even though they won't make
|
||||
// the top-5 recent). The bounded window keeps scan cost
|
||||
// linear in file size — we're reading tail only.
|
||||
let outcome_rows = tail_matching(
|
||||
outcomes_path, AGGREGATE_WINDOW * 4,
|
||||
|row| {
|
||||
let row_sig = row.get("sig_hash").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let row_tc = row.get("task_class").and_then(|v| v.as_str()).unwrap_or("");
|
||||
row_sig == sig_hash || row_tc == task_class
|
||||
},
|
||||
).await;
|
||||
|
||||
// Recent outcomes: exact sig_hash match first (strongest
|
||||
// signal), then task_class fallback up to the limit.
|
||||
let mut exact: Vec<OutcomeSummary> = Vec::new();
|
||||
let mut loose: Vec<OutcomeSummary> = Vec::new();
|
||||
for row in &outcome_rows {
|
||||
let row_sig = row.get("sig_hash").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let summary = summarize_outcome(row);
|
||||
if row_sig == sig_hash { exact.push(summary); }
|
||||
else { loose.push(summary); }
|
||||
}
|
||||
ctx.recent_outcomes = exact.into_iter().rev().take(RECENT_OUTCOME_LIMIT).collect();
|
||||
if ctx.recent_outcomes.len() < RECENT_OUTCOME_LIMIT {
|
||||
let need = RECENT_OUTCOME_LIMIT - ctx.recent_outcomes.len();
|
||||
ctx.recent_outcomes.extend(loose.into_iter().rev().take(need));
|
||||
}
|
||||
|
||||
// Aggregate rates across the full matched window (both
|
||||
// sig_hash and task_class matches — gives a stable rate even
|
||||
// on sparse sig_hash history).
|
||||
let window = outcome_rows.iter().rev().take(AGGREGATE_WINDOW);
|
||||
let mut ok_count = 0u32;
|
||||
let mut total = 0u32;
|
||||
let mut turn_sum = 0u32;
|
||||
let mut latency_sum = 0u64;
|
||||
for row in window {
|
||||
total += 1;
|
||||
if row.get("ok").and_then(|v| v.as_bool()).unwrap_or(false) { ok_count += 1; }
|
||||
turn_sum += row.get("turns").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
||||
latency_sum += row.get("usage")
|
||||
.and_then(|u| u.get("latency_ms"))
|
||||
.and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
}
|
||||
if total > 0 {
|
||||
ctx.total_observed = total;
|
||||
ctx.success_rate = Some(ok_count as f64 / total as f64);
|
||||
ctx.avg_turns = Some(turn_sum as f64 / total as f64);
|
||||
ctx.avg_latency_ms = Some(latency_sum / total as u64);
|
||||
}
|
||||
|
||||
// Overseer corrections. Prefer sig_hash match; fall back to
|
||||
// task_class. The overseer reading its OWN prior corrections
|
||||
// is the main point — if the last 3 attempts produced
|
||||
// corrections X, Y, Z, the new correction should acknowledge
|
||||
// those patterns rather than suggest X for the fourth time.
|
||||
let correction_rows = tail_matching(
|
||||
corrections_path, RECENT_CORRECTION_LIMIT * 4,
|
||||
|row| {
|
||||
let row_sig = row.get("sig_hash").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let row_tc = row.get("task_class").and_then(|v| v.as_str()).unwrap_or("");
|
||||
row_sig == sig_hash || row_tc == task_class
|
||||
},
|
||||
).await;
|
||||
let mut c_exact: Vec<CorrectionSummary> = Vec::new();
|
||||
let mut c_loose: Vec<CorrectionSummary> = Vec::new();
|
||||
for row in &correction_rows {
|
||||
let row_sig = row.get("sig_hash").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let summary = summarize_correction(row);
|
||||
if row_sig == sig_hash { c_exact.push(summary); }
|
||||
else { c_loose.push(summary); }
|
||||
}
|
||||
ctx.recent_corrections = c_exact.into_iter().rev().take(RECENT_CORRECTION_LIMIT).collect();
|
||||
if ctx.recent_corrections.len() < RECENT_CORRECTION_LIMIT {
|
||||
let need = RECENT_CORRECTION_LIMIT - ctx.recent_corrections.len();
|
||||
ctx.recent_corrections.extend(c_loose.into_iter().rev().take(need));
|
||||
}
|
||||
|
||||
ctx
|
||||
}
|
||||
|
||||
/// Compact string form for the overseer prompt. Deterministic
|
||||
/// ordering + bounded length so prompt caching stays stable
|
||||
/// across iterations on the same task.
|
||||
pub fn to_prompt_section(&self) -> String {
|
||||
let mut s = String::new();
|
||||
s.push_str("## Knowledge Base Context\n");
|
||||
if let (Some(rate), Some(turns), Some(lat)) = (self.success_rate, self.avg_turns, self.avg_latency_ms) {
|
||||
s.push_str(&format!(
|
||||
"Across {} prior similar runs: success_rate={:.1}%, avg_turns={:.1}, avg_latency_ms={}\n",
|
||||
self.total_observed, rate * 100.0, turns, lat,
|
||||
));
|
||||
} else {
|
||||
s.push_str("No prior similar runs recorded.\n");
|
||||
}
|
||||
|
||||
if !self.recent_outcomes.is_empty() {
|
||||
s.push_str(&format!("\nRecent {} outcomes:\n", self.recent_outcomes.len()));
|
||||
for o in &self.recent_outcomes {
|
||||
let err = o.error.as_deref().map(|e| format!(" — {}", truncate(e, 80))).unwrap_or_default();
|
||||
s.push_str(&format!(
|
||||
" [{}] ok={} turns={} tokens={} lat={}ms{}\n",
|
||||
&o.created_at[..19.min(o.created_at.len())],
|
||||
o.ok, o.turns, o.total_tokens, o.latency_ms, err,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if !self.recent_corrections.is_empty() {
|
||||
s.push_str(&format!("\nRecent {} overseer corrections (yours — don't repeat):\n", self.recent_corrections.len()));
|
||||
for c in &self.recent_corrections {
|
||||
s.push_str(&format!(
|
||||
" [{}] turn={} reason={} correction={}\n",
|
||||
&c.created_at[..19.min(c.created_at.len())],
|
||||
c.applied_at_turn,
|
||||
truncate(&c.reason, 40),
|
||||
truncate(&c.correction_preview, 200),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_outcome(row: &serde_json::Value) -> OutcomeSummary {
|
||||
OutcomeSummary {
|
||||
created_at: row.get("created_at").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
||||
ok: row.get("ok").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
polarity: row.get("polarity").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
||||
turns: row.get("turns").and_then(|v| v.as_u64()).unwrap_or(0) as u32,
|
||||
latency_ms: row.get("usage").and_then(|u| u.get("latency_ms"))
|
||||
.and_then(|v| v.as_u64()).unwrap_or(0),
|
||||
total_tokens: row.get("usage").and_then(|u| u.get("total_tokens"))
|
||||
.and_then(|v| v.as_u64()).unwrap_or(0),
|
||||
error: row.get("error").and_then(|v| v.as_str()).map(String::from),
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_correction(row: &serde_json::Value) -> CorrectionSummary {
|
||||
let preview = row.get("correction").and_then(|v| v.as_str()).unwrap_or("");
|
||||
CorrectionSummary {
|
||||
created_at: row.get("created_at").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
||||
reason: row.get("reason").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
||||
correction_preview: truncate(preview, 300),
|
||||
applied_at_turn: row.get("applied_at_turn").and_then(|v| v.as_u64()).unwrap_or(0) as u32,
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate(s: &str, n: usize) -> String {
|
||||
if s.len() <= n { s.to_string() } else { format!("{}…", &s[..n]) }
|
||||
}
|
||||
|
||||
/// Read a JSONL file from the tail, returning at most `limit` rows
|
||||
/// that match `filter`. Missing file returns empty. Corrupt lines are
|
||||
/// skipped. Limit is honored from the tail — a full-file scan with an
|
||||
/// in-memory ring would be wasteful for large outcomes histories, but
|
||||
/// we cap at reading the whole file and filtering post-hoc for now
|
||||
/// (reverse-seek line iteration is a real engineering task and the
|
||||
/// file is bounded by ingest rate; revisit when it bites).
|
||||
async fn tail_matching<F>(
|
||||
path: &Path,
|
||||
limit: usize,
|
||||
filter: F,
|
||||
) -> Vec<serde_json::Value>
|
||||
where
|
||||
F: Fn(&serde_json::Value) -> bool,
|
||||
{
|
||||
let Ok(file) = tokio::fs::File::open(path).await else { return Vec::new(); };
|
||||
let reader = tokio::io::BufReader::new(file);
|
||||
let mut lines = reader.lines();
|
||||
let mut matches: Vec<serde_json::Value> = Vec::new();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(&line) else { continue };
|
||||
if filter(&v) {
|
||||
matches.push(v);
|
||||
if matches.len() > limit {
|
||||
// Keep the most-recent window only — drop from the
|
||||
// front as we go rather than buffering everything.
|
||||
matches.remove(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
matches
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
async fn write_fixture(path: &Path, rows: Vec<serde_json::Value>) {
|
||||
if let Some(dir) = path.parent() {
|
||||
tokio::fs::create_dir_all(dir).await.unwrap();
|
||||
}
|
||||
let mut f = tokio::fs::OpenOptions::new()
|
||||
.create(true).write(true).truncate(true).open(path).await.unwrap();
|
||||
for r in rows {
|
||||
let mut line = serde_json::to_string(&r).unwrap();
|
||||
line.push('\n');
|
||||
f.write_all(line.as_bytes()).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn tmp_path(name: &str) -> std::path::PathBuf {
|
||||
let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos();
|
||||
std::env::temp_dir().join(format!("lh_kb_ctx_{}_{}_{}", std::process::id(), nanos, name))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_files_produce_empty_context() {
|
||||
let op = tmp_path("outcomes.jsonl");
|
||||
let cp = tmp_path("corrections.jsonl");
|
||||
let ctx = KbContext::load_from("sig123", "staffing.fill", &op, &cp).await;
|
||||
assert!(ctx.recent_outcomes.is_empty());
|
||||
assert!(ctx.recent_corrections.is_empty());
|
||||
assert!(ctx.success_rate.is_none());
|
||||
assert_eq!(ctx.total_observed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_sig_hash_matches_take_priority() {
|
||||
let op = tmp_path("outcomes.jsonl");
|
||||
let cp = tmp_path("corrections.jsonl");
|
||||
write_fixture(&op, vec![
|
||||
// Other sig_hash, same task_class — loose match
|
||||
serde_json::json!({
|
||||
"sig_hash": "other", "task_class": "staffing.fill",
|
||||
"ok": false, "polarity": "failure_pattern", "turns": 1,
|
||||
"usage": {"latency_ms": 1000, "total_tokens": 100},
|
||||
"created_at": "2026-04-22T10:00:00Z",
|
||||
}),
|
||||
// Exact sig_hash — should lead
|
||||
serde_json::json!({
|
||||
"sig_hash": "sig123", "task_class": "staffing.fill",
|
||||
"ok": true, "polarity": "success_confirmation", "turns": 3,
|
||||
"usage": {"latency_ms": 2000, "total_tokens": 500},
|
||||
"created_at": "2026-04-23T10:00:00Z",
|
||||
}),
|
||||
]).await;
|
||||
write_fixture(&cp, vec![]).await;
|
||||
|
||||
let ctx = KbContext::load_from("sig123", "staffing.fill", &op, &cp).await;
|
||||
assert_eq!(ctx.recent_outcomes.len(), 2);
|
||||
assert_eq!(ctx.recent_outcomes[0].created_at, "2026-04-23T10:00:00Z");
|
||||
assert_eq!(ctx.recent_outcomes[0].ok, true);
|
||||
assert_eq!(ctx.total_observed, 2);
|
||||
assert!((ctx.success_rate.unwrap() - 0.5).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_rows_are_skipped() {
|
||||
let op = tmp_path("outcomes.jsonl");
|
||||
let cp = tmp_path("corrections.jsonl");
|
||||
// Mix valid + invalid — invalid should be silently skipped.
|
||||
if let Some(dir) = op.parent() { tokio::fs::create_dir_all(dir).await.unwrap(); }
|
||||
tokio::fs::write(&op, "not json\n{\"sig_hash\":\"sig1\",\"task_class\":\"tc\",\"ok\":true,\"turns\":1,\"usage\":{}}\ngarbage\n").await.unwrap();
|
||||
write_fixture(&cp, vec![]).await;
|
||||
let ctx = KbContext::load_from("sig1", "tc", &op, &cp).await;
|
||||
assert_eq!(ctx.recent_outcomes.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrections_preview_is_truncated() {
|
||||
let op = tmp_path("outcomes.jsonl");
|
||||
let cp = tmp_path("corrections.jsonl");
|
||||
let long = "x".repeat(500);
|
||||
write_fixture(&op, vec![]).await;
|
||||
write_fixture(&cp, vec![serde_json::json!({
|
||||
"sig_hash": "sig1", "task_class": "tc",
|
||||
"reason": "abort", "correction": long, "applied_at_turn": 3,
|
||||
"created_at": "2026-04-23T10:00:00Z",
|
||||
})]).await;
|
||||
let ctx = KbContext::load_from("sig1", "tc", &op, &cp).await;
|
||||
assert_eq!(ctx.recent_corrections.len(), 1);
|
||||
// 300-char cap + 3-byte UTF-8 ellipsis character = 303-byte worst case.
|
||||
assert!(ctx.recent_corrections[0].correction_preview.len() <= 303);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_section_is_stable_for_empty_context() {
|
||||
let ctx = KbContext::default();
|
||||
let s = ctx.to_prompt_section();
|
||||
assert!(s.contains("No prior similar runs recorded"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_section_reports_aggregate_rates() {
|
||||
let ctx = KbContext {
|
||||
total_observed: 10,
|
||||
success_rate: Some(0.7),
|
||||
avg_turns: Some(4.2),
|
||||
avg_latency_ms: Some(45000),
|
||||
..Default::default()
|
||||
};
|
||||
let s = ctx.to_prompt_section();
|
||||
assert!(s.contains("success_rate=70.0%"));
|
||||
assert!(s.contains("avg_turns=4.2"));
|
||||
assert!(s.contains("avg_latency_ms=45000"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,345 @@
|
||||
mod access;
|
||||
mod access_service;
|
||||
mod auth;
|
||||
mod execution_loop;
|
||||
mod observability;
|
||||
mod tools;
|
||||
mod v1;
|
||||
|
||||
use axum::{Router, extract::DefaultBodyLimit, routing::get};
|
||||
use proto::lakehouse::catalog_service_server::CatalogServiceServer;
|
||||
use shared::config::Config;
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
// Load config
|
||||
let config = Config::load_or_default();
|
||||
|
||||
// Initialize tracing + observability
|
||||
observability::init_tracing(
|
||||
&config.observability.service_name,
|
||||
&config.observability.exporter,
|
||||
);
|
||||
|
||||
tracing::info!("config loaded: gateway={}:{}, storage={}",
|
||||
config.gateway.host, config.gateway.port, config.storage.root);
|
||||
|
||||
// Secrets provider
|
||||
let secrets = std::sync::Arc::new(shared::secrets::FileSecretsProvider::from_env());
|
||||
if let Err(e) = secrets.load().await {
|
||||
tracing::warn!("secrets load: {e} — bucket credentials may be missing");
|
||||
}
|
||||
|
||||
// Federation: bucket registry (primary + any [[storage.buckets]] entries + rescue)
|
||||
let bucket_registry = std::sync::Arc::new(
|
||||
storaged::registry::BucketRegistry::from_config(&config.storage, secrets.clone())
|
||||
.await
|
||||
.expect("bucket registry init failed")
|
||||
);
|
||||
let buckets = bucket_registry.list().await;
|
||||
tracing::info!(
|
||||
"bucket registry: {} buckets [{}], rescue={:?}",
|
||||
buckets.len(),
|
||||
buckets.iter().map(|b| format!("{}:{}", b.name, b.backend)).collect::<Vec<_>>().join(", "),
|
||||
bucket_registry.rescue_name(),
|
||||
);
|
||||
|
||||
// Back-compat: existing components that expect a single Arc<ObjectStore>
|
||||
// get the primary bucket's store. As we migrate call sites to be bucket-
|
||||
// aware, they'll call bucket_registry.get(name) directly.
|
||||
let store = bucket_registry.default_store();
|
||||
|
||||
// Catalog
|
||||
let registry = catalogd::registry::Registry::new(store.clone());
|
||||
if let Err(e) = registry.rebuild().await {
|
||||
tracing::warn!("catalog rebuild failed (empty store?): {e}");
|
||||
}
|
||||
|
||||
// Query engine with 16GB memory cache
|
||||
let cache = queryd::cache::MemCache::new(16 * 1024 * 1024 * 1024);
|
||||
let engine = queryd::context::QueryEngine::new(registry.clone(), bucket_registry.clone(), cache);
|
||||
|
||||
// Event journal — append-only mutation log (flush every 100 events)
|
||||
let journal = journald::journal::Journal::new(store.clone(), 100);
|
||||
|
||||
// Access control
|
||||
let access = access::AccessControl::new(config.auth.enabled);
|
||||
access.register_defaults().await;
|
||||
|
||||
// Phase 42 — file-backed truth rules. Probes the `truth/` directory
|
||||
// at repo root (or $LAKEHOUSE_TRUTH_DIR override) and logs how many
|
||||
// rules load. Current request paths still build their own stores
|
||||
// via truth::default_truth_store() / truth::sql_query_guard_store();
|
||||
// the composed-at-boot store gets plumbed through V1State in a
|
||||
// follow-up. This boot probe catches parse errors + duplicate-ID
|
||||
// collisions early rather than at first request.
|
||||
{
|
||||
let truth_dir = std::env::var("LAKEHOUSE_TRUTH_DIR")
|
||||
.unwrap_or_else(|_| "/home/profit/lakehouse/truth".to_string());
|
||||
if std::path::Path::new(&truth_dir).exists() {
|
||||
let mut probe_store = truth::default_truth_store();
|
||||
match truth::loader::load_from_dir(&mut probe_store, &truth_dir) {
|
||||
Ok(n) => tracing::info!("truth: loaded {n} file-backed rule(s) from {truth_dir}"),
|
||||
Err(e) => tracing::warn!("truth: failed to load rules from {truth_dir}: {e}"),
|
||||
}
|
||||
} else {
|
||||
tracing::debug!("truth: no rule dir at {truth_dir}, skipping file-backed load");
|
||||
}
|
||||
}
|
||||
|
||||
// Workspace manager for agent-specific overlays
|
||||
let workspace_mgr = queryd::workspace::WorkspaceManager::new(store.clone());
|
||||
if let Err(e) = workspace_mgr.rebuild().await {
|
||||
tracing::warn!("workspace rebuild: {e}");
|
||||
}
|
||||
|
||||
// AI sidecar client
|
||||
let ai_client = aibridge::client::AiClient::new(&config.sidecar.url);
|
||||
|
||||
// Vector service components — built before the router because both the
|
||||
// /vectors service AND ingestd need the agent handle to enqueue triggers.
|
||||
let index_reg = vectord::index_registry::IndexRegistry::new(store.clone());
|
||||
let _ = index_reg.rebuild().await;
|
||||
let hnsw = vectord::hnsw::HnswStore::new();
|
||||
let emb_cache = vectord::embedding_cache::EmbeddingCache::new(store.clone());
|
||||
// Phase B (federation layer 2): trial journals + promotion files now
|
||||
// live in each index's recorded bucket (IndexMeta.bucket), not always
|
||||
// primary. Journal / registry resolve per-call via the bucket registry.
|
||||
let tj = vectord::trial::TrialJournal::new(bucket_registry.clone(), index_reg.clone());
|
||||
let pr = vectord::promotion::PromotionRegistry::new(bucket_registry.clone(), index_reg.clone());
|
||||
let hs = vectord::harness::HarnessStore::new(bucket_registry.clone(), index_reg.clone());
|
||||
// Phase 19: playbook memory. Load cached state; empty-on-miss is fine,
|
||||
// operators call POST /vectors/playbook_memory/rebuild to populate.
|
||||
let pbm = vectord::playbook_memory::PlaybookMemory::new(store.clone());
|
||||
let _ = pbm.load_from_storage().await;
|
||||
// Pathway memory — consensus-designed sidecar for full-context
|
||||
// backtracking + hot-swap of successful review pathways. Same
|
||||
// load-on-boot pattern as playbook_memory: empty state is fine,
|
||||
// operators start populating via scrum_master_pipeline.ts.
|
||||
let pwm = vectord::pathway_memory::PathwayMemory::new(store.clone());
|
||||
let _ = pwm.load_from_storage().await;
|
||||
|
||||
// Phase 16.2: spawn the autotune agent. When config.agent.enabled=false
|
||||
// this returns a handle that drops triggers silently — no surprise load.
|
||||
let agent_cfg = vectord::agent::AgentConfig {
|
||||
enabled: config.agent.enabled,
|
||||
cycle_interval_secs: config.agent.cycle_interval_secs,
|
||||
cooldown_between_trials_secs: config.agent.cooldown_between_trials_secs,
|
||||
min_recall: config.agent.min_recall,
|
||||
max_trials_per_hour: config.agent.max_trials_per_hour,
|
||||
};
|
||||
let agent_handle = vectord::agent::spawn(
|
||||
agent_cfg,
|
||||
vectord::agent::AgentDeps {
|
||||
store: store.clone(),
|
||||
ai_client: ai_client.clone(),
|
||||
catalog: registry.clone(),
|
||||
index_registry: index_reg.clone(),
|
||||
hnsw_store: hnsw.clone(),
|
||||
embedding_cache: emb_cache.clone(),
|
||||
trial_journal: tj.clone(),
|
||||
promotion_registry: pr.clone(),
|
||||
harness_store: hs.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
// HTTP router
|
||||
let mut app = Router::new()
|
||||
.route("/health", get(health))
|
||||
.nest("/storage", storaged::service::router(store.clone())
|
||||
.merge(storaged::federation_service::router(bucket_registry.clone())))
|
||||
.nest("/catalog", catalogd::service::router(registry.clone()))
|
||||
.nest("/query", queryd::service::router(engine.clone()))
|
||||
.nest("/ai", aibridge::service::router(ai_client.clone()))
|
||||
;
|
||||
|
||||
// Phase E: scheduled ingest
|
||||
let sched_store = ingestd::schedule::ScheduleStore::new(store.clone());
|
||||
if let Ok(n) = sched_store.rebuild().await {
|
||||
if n > 0 { tracing::info!("rebuilt {n} persisted schedule(s)"); }
|
||||
}
|
||||
let scheduler = ingestd::schedule::Scheduler {
|
||||
store: sched_store.clone(),
|
||||
ingest: ingestd::schedule::SchedulerIngestDeps {
|
||||
store: store.clone(),
|
||||
registry: registry.clone(),
|
||||
buckets: bucket_registry.clone(),
|
||||
agent_handle: Some(agent_handle.clone()),
|
||||
index_registry: index_reg.clone(),
|
||||
},
|
||||
tick_secs: 10,
|
||||
};
|
||||
scheduler.spawn();
|
||||
|
||||
app = app
|
||||
.nest("/ingest", ingestd::service::router(ingestd::service::IngestState {
|
||||
store: store.clone(),
|
||||
registry: registry.clone(),
|
||||
buckets: bucket_registry.clone(),
|
||||
agent_handle: agent_handle.clone(),
|
||||
index_registry: index_reg.clone(),
|
||||
schedules: sched_store,
|
||||
// P9-001 fix 2026-04-23: journal reference flows into ingest so
|
||||
// successful uploads emit a record_ingest event. Journal is Clone
|
||||
// (Arc<RwLock> inside) so the /journal nest below still sees the
|
||||
// same buffer + persistence.
|
||||
journal: Some(journal.clone()),
|
||||
}))
|
||||
.nest("/vectors", vectord::service::router(vectord::service::VectorState {
|
||||
store: store.clone(),
|
||||
ai_client: ai_client.clone(),
|
||||
job_tracker: vectord::jobs::JobTracker::new(),
|
||||
index_registry: index_reg.clone(),
|
||||
hnsw_store: hnsw,
|
||||
embedding_cache: emb_cache,
|
||||
trial_journal: tj,
|
||||
harness_store: hs,
|
||||
catalog: registry.clone(),
|
||||
promotion_registry: pr,
|
||||
agent_handle,
|
||||
bucket_registry: bucket_registry.clone(),
|
||||
active_profile: std::sync::Arc::new(tokio::sync::RwLock::new(None)),
|
||||
lance: vectord::lance_backend::LanceRegistry::new(
|
||||
bucket_registry.clone(), index_reg.clone(),
|
||||
),
|
||||
playbook_memory: pbm,
|
||||
pathway_memory: pwm,
|
||||
embed_semaphore: std::sync::Arc::new(tokio::sync::Semaphore::new(1)),
|
||||
}))
|
||||
.nest("/workspaces", queryd::workspace_service::router(workspace_mgr))
|
||||
.nest("/journal", journald::service::router(journal))
|
||||
.nest("/access", access_service::router(access))
|
||||
.nest("/tools", tools::service::router({
|
||||
let tool_reg = tools::registry::ToolRegistry::new();
|
||||
tool_reg.register_defaults().await;
|
||||
tools::ToolState {
|
||||
registry: tool_reg,
|
||||
query_fn: tools::QueryExecutor::new(engine.clone()),
|
||||
truth: std::sync::Arc::new(truth::sql_query_guard_store()),
|
||||
}
|
||||
}))
|
||||
// Phase 38 — Universal API skeleton. Thin OpenAI-compatible
|
||||
// surface over the existing aibridge → Ollama path. Future
|
||||
// phases add provider adapters (39), routing engine (40),
|
||||
// session state (41), etc. All without changing this mount.
|
||||
.nest("/v1", v1::router(v1::V1State {
|
||||
ai_client: ai_client.clone(),
|
||||
usage: std::sync::Arc::new(tokio::sync::RwLock::new(v1::Usage::default())),
|
||||
// Phase 39 first slice — Ollama Cloud adapter. Key resolved
|
||||
// from OLLAMA_CLOUD_KEY env, then /root/llm_team_config.json,
|
||||
// then OLLAMA_CLOUD_API_KEY env. None = cloud routes 503.
|
||||
ollama_cloud_key: {
|
||||
let k = v1::ollama_cloud::resolve_cloud_key();
|
||||
if k.is_some() {
|
||||
tracing::info!("v1: Ollama Cloud key loaded — /v1/chat provider=ollama_cloud enabled");
|
||||
} else {
|
||||
tracing::warn!("v1: no Ollama Cloud key in env or /root/llm_team_config.json — cloud routes will 503");
|
||||
}
|
||||
k
|
||||
},
|
||||
openrouter_key: {
|
||||
// 2026-04-24 free-tier rescue rung for iter 5+. Shares
|
||||
// the LLM Team UI's OPENROUTER_API_KEY so both systems
|
||||
// draw from one quota.
|
||||
let k = v1::openrouter::resolve_openrouter_key();
|
||||
if k.is_some() {
|
||||
tracing::info!("v1: OpenRouter key loaded — /v1/chat provider=openrouter enabled");
|
||||
} else {
|
||||
tracing::warn!("v1: no OpenRouter key — openrouter rescue rung will 503");
|
||||
}
|
||||
k
|
||||
},
|
||||
gemini_key: {
|
||||
// Phase 40 provider. GEMINI_API_KEY in env or .env.
|
||||
let k = v1::gemini::resolve_gemini_key();
|
||||
if k.is_some() {
|
||||
tracing::info!("v1: Gemini key loaded — /v1/chat provider=gemini enabled");
|
||||
} else {
|
||||
tracing::debug!("v1: no Gemini key — provider=gemini will 503");
|
||||
}
|
||||
k
|
||||
},
|
||||
claude_key: {
|
||||
// Phase 40 provider. ANTHROPIC_API_KEY in env or .env.
|
||||
let k = v1::claude::resolve_claude_key();
|
||||
if k.is_some() {
|
||||
tracing::info!("v1: Claude key loaded — /v1/chat provider=claude enabled");
|
||||
} else {
|
||||
tracing::debug!("v1: no Claude key — provider=claude will 503");
|
||||
}
|
||||
k
|
||||
},
|
||||
// Phase 40 early deliverable — Langfuse trace emitter.
|
||||
// Defaults match mcp-server/tracing.ts conventions so
|
||||
// gateway traces land in the same staffing project.
|
||||
langfuse: {
|
||||
let c = v1::langfuse_trace::LangfuseClient::from_env_or_defaults();
|
||||
if c.is_some() {
|
||||
tracing::info!("v1: Langfuse tracing enabled — /v1/chat calls will appear at localhost:3001");
|
||||
} else {
|
||||
tracing::warn!("v1: Langfuse keys missing — /v1/chat calls will not be traced");
|
||||
}
|
||||
c
|
||||
},
|
||||
}));
|
||||
|
||||
// Auth middleware (if enabled) — P5-001 fix 2026-04-23:
|
||||
// previously only inserted the ApiKey as an extension and never layered
|
||||
// the middleware, so auth.enabled=true enforced nothing. Now wraps the
|
||||
// router with from_fn_with_state, which calls api_key_auth on every
|
||||
// request. /health is exempted inside the middleware (LB probes).
|
||||
if config.auth.enabled {
|
||||
if let Some(ref key) = config.auth.api_key {
|
||||
tracing::info!("API key auth enabled — enforcing on all routes except /health");
|
||||
let api_key = auth::ApiKey(key.clone());
|
||||
app = app.layer(axum::middleware::from_fn_with_state(
|
||||
api_key,
|
||||
auth::api_key_auth,
|
||||
));
|
||||
} else {
|
||||
tracing::warn!("auth enabled but no api_key set — all requests allowed");
|
||||
}
|
||||
}
|
||||
|
||||
app = app
|
||||
.layer(DefaultBodyLimit::max(512 * 1024 * 1024)) // 512MB — supports 500K worker CSV
|
||||
.layer(CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_methods(Any)
|
||||
.allow_headers(Any))
|
||||
.layer(TraceLayer::new_for_http());
|
||||
|
||||
// File watcher — auto-ingest from ./inbox
|
||||
let _watcher = ingestd::watcher::spawn_watcher(
|
||||
ingestd::watcher::WatchConfig::default(),
|
||||
store.clone(),
|
||||
registry.clone(),
|
||||
);
|
||||
|
||||
// Start gRPC server on port+1
|
||||
let grpc_port = config.gateway.port + 1;
|
||||
let catalog_grpc = catalogd::grpc::CatalogGrpc::new(registry);
|
||||
let grpc_addr = format!("{}:{}", config.gateway.host, grpc_port).parse().unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
tracing::info!("gRPC server listening on {grpc_addr}");
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(CatalogServiceServer::new(catalog_grpc))
|
||||
.serve(grpc_addr)
|
||||
.await
|
||||
.expect("gRPC server failed");
|
||||
});
|
||||
|
||||
// Start HTTP server
|
||||
let http_addr = format!("{}:{}", config.gateway.host, config.gateway.port);
|
||||
tracing::info!("HTTP gateway listening on {http_addr}");
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&http_addr).await.unwrap();
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
}
|
||||
|
||||
async fn health() -> &'static str {
|
||||
"lakehouse ok"
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
use opentelemetry::trace::TracerProvider;
|
||||
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||
use tracing_opentelemetry::OpenTelemetryLayer;
|
||||
use tracing_subscriber::{EnvFilter, Registry, fmt, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
/// Initialize tracing with optional OpenTelemetry export.
|
||||
pub fn init_tracing(service_name: &str, exporter: &str) {
|
||||
let env_filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
|
||||
match exporter {
|
||||
"stdout" => {
|
||||
// OTel traces to stdout + regular fmt layer
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||
.build();
|
||||
let tracer = provider.tracer(service_name.to_string());
|
||||
|
||||
Registry::default()
|
||||
.with(env_filter)
|
||||
.with(fmt::layer())
|
||||
.with(OpenTelemetryLayer::new(tracer))
|
||||
.init();
|
||||
|
||||
tracing::info!("tracing initialized: fmt + opentelemetry-stdout");
|
||||
}
|
||||
_ => {
|
||||
// Just fmt layer, no OTel
|
||||
Registry::default()
|
||||
.with(env_filter)
|
||||
.with(fmt::layer())
|
||||
.init();
|
||||
|
||||
tracing::info!("tracing initialized: fmt only");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
pub mod registry;
|
||||
pub mod service;
|
||||
|
||||
use queryd::context::QueryEngine;
|
||||
use arrow::json::writer::{JsonArray, Writer as JsonWriter};
|
||||
use std::sync::Arc;
|
||||
use truth::TruthStore;
|
||||
|
||||
/// State for the tool system.
|
||||
#[derive(Clone)]
|
||||
pub struct ToolState {
|
||||
pub registry: registry::ToolRegistry,
|
||||
pub query_fn: QueryExecutor,
|
||||
/// SQL guard (shared with queryd). Mirrors the queryd /sql truth
|
||||
/// gate from P42-002 (9cc0ceb) — tools also execute model-
|
||||
/// originated SQL, need the same destructive-verb block.
|
||||
pub truth: Arc<TruthStore>,
|
||||
}
|
||||
|
||||
/// Wraps QueryEngine to provide a simple execute interface for tools.
|
||||
#[derive(Clone)]
|
||||
pub struct QueryExecutor {
|
||||
engine: QueryEngine,
|
||||
}
|
||||
|
||||
impl QueryExecutor {
|
||||
pub fn new(engine: QueryEngine) -> Self {
|
||||
Self { engine }
|
||||
}
|
||||
|
||||
/// Execute SQL and return (rows as JSON, row count).
|
||||
pub async fn execute(&self, sql: &str) -> Result<(serde_json::Value, usize), String> {
|
||||
let batches = self.engine.query(sql).await?;
|
||||
|
||||
if batches.is_empty() {
|
||||
return Ok((serde_json::Value::Array(vec![]), 0));
|
||||
}
|
||||
|
||||
let mut buf = Vec::new();
|
||||
let mut writer = JsonWriter::<_, JsonArray>::new(&mut buf);
|
||||
for batch in &batches {
|
||||
writer.write(batch).map_err(|e| format!("JSON write: {e}"))?;
|
||||
}
|
||||
writer.finish().map_err(|e| format!("JSON finish: {e}"))?;
|
||||
drop(writer);
|
||||
|
||||
let rows: serde_json::Value = serde_json::from_slice(&buf)
|
||||
.map_err(|e| format!("JSON parse: {e}"))?;
|
||||
let count = rows.as_array().map(|a| a.len()).unwrap_or(0);
|
||||
|
||||
Ok((rows, count))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
/// Tool Registry: named, governed business actions for AI agents.
|
||||
/// Instead of raw SQL, agents call validated tools with audit trails.
|
||||
///
|
||||
/// Each tool has:
|
||||
/// - Name and description (for LLM tool-use)
|
||||
/// - Parameter schema (validated before execution)
|
||||
/// - Permission level (read / write / admin)
|
||||
/// - Audit logging (every invocation recorded)
|
||||
/// - Rate limiting (per agent)
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Permission level for a tool.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Permission {
|
||||
Read, // auto-approved, no side effects
|
||||
Write, // modifies data, logged
|
||||
Admin, // destructive, requires confirmation
|
||||
}
|
||||
|
||||
/// Parameter definition for a tool.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ParamDef {
|
||||
pub name: String,
|
||||
pub param_type: String, // "string", "integer", "boolean", "float"
|
||||
pub required: bool,
|
||||
pub description: String,
|
||||
pub default: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Tool definition — what agents see and can call.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ToolDef {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub permission: Permission,
|
||||
pub parameters: Vec<ParamDef>,
|
||||
pub returns: String, // description of return value
|
||||
pub sql_template: String, // SQL with {param} placeholders
|
||||
pub category: String, // "candidates", "placements", "analytics"
|
||||
}
|
||||
|
||||
/// Audit log entry for a tool invocation.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ToolInvocation {
|
||||
pub id: String,
|
||||
pub tool_name: String,
|
||||
pub agent: String,
|
||||
pub params: serde_json::Value,
|
||||
pub permission: Permission,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub success: bool,
|
||||
pub error: Option<String>,
|
||||
pub rows_returned: Option<usize>,
|
||||
}
|
||||
|
||||
/// The registry — holds tool definitions and audit log.
|
||||
#[derive(Clone)]
|
||||
pub struct ToolRegistry {
|
||||
tools: Arc<RwLock<HashMap<String, ToolDef>>>,
|
||||
audit_log: Arc<RwLock<Vec<ToolInvocation>>>,
|
||||
}
|
||||
|
||||
impl ToolRegistry {
|
||||
/// Build an empty registry. Callers in an async context should follow
|
||||
/// this with `.register_defaults().await` if they want the built-in
|
||||
/// staffing tools pre-installed — main.rs does exactly that.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tools: Arc::new(RwLock::new(HashMap::new())),
|
||||
audit_log: Arc::new(RwLock::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register default staffing tools.
|
||||
pub async fn register_defaults(&self) {
|
||||
let tools = vec![
|
||||
ToolDef {
|
||||
name: "search_candidates".into(),
|
||||
description: "Search for candidates by skills, city, state, availability, and experience. Returns matching candidates with contact info.".into(),
|
||||
permission: Permission::Read,
|
||||
parameters: vec![
|
||||
ParamDef { name: "skills".into(), param_type: "string".into(), required: false, description: "Comma-separated skills to match (uses LIKE)".into(), default: None },
|
||||
ParamDef { name: "city".into(), param_type: "string".into(), required: false, description: "City name".into(), default: None },
|
||||
ParamDef { name: "state".into(), param_type: "string".into(), required: false, description: "State abbreviation".into(), default: None },
|
||||
ParamDef { name: "min_years".into(), param_type: "integer".into(), required: false, description: "Minimum years of experience".into(), default: Some(serde_json::json!(0)) },
|
||||
ParamDef { name: "status".into(), param_type: "string".into(), required: false, description: "Candidate status (active, inactive, placed)".into(), default: Some(serde_json::json!("active")) },
|
||||
ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
|
||||
],
|
||||
returns: "List of candidates with id, name, phone, email, skills, experience".into(),
|
||||
sql_template: "SELECT candidate_id, first_name, last_name, phone, email, city, state, skills, years_experience, status FROM candidates WHERE 1=1 {skills_filter} {city_filter} {state_filter} {years_filter} {status_filter} ORDER BY years_experience DESC LIMIT {limit}".into(),
|
||||
category: "candidates".into(),
|
||||
},
|
||||
ToolDef {
|
||||
name: "get_candidate".into(),
|
||||
description: "Get full details for a specific candidate by ID.".into(),
|
||||
permission: Permission::Read,
|
||||
parameters: vec![
|
||||
ParamDef { name: "candidate_id".into(), param_type: "string".into(), required: true, description: "Candidate ID (e.g. CAND-000001)".into(), default: None },
|
||||
],
|
||||
returns: "Full candidate record".into(),
|
||||
sql_template: "SELECT * FROM candidates WHERE candidate_id = '{candidate_id}'".into(),
|
||||
category: "candidates".into(),
|
||||
},
|
||||
ToolDef {
|
||||
name: "revenue_by_client".into(),
|
||||
description: "Show total billed revenue, pay costs, and gross profit by client. Filter by date range.".into(),
|
||||
permission: Permission::Read,
|
||||
parameters: vec![
|
||||
ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Top N clients".into(), default: Some(serde_json::json!(10)) },
|
||||
],
|
||||
returns: "Client name, total billed, total paid, gross profit, timesheet count".into(),
|
||||
sql_template: "SELECT c.company_name, COUNT(*) as timesheets, ROUND(SUM(t.bill_total),2) as total_billed, ROUND(SUM(t.pay_total),2) as total_paid, ROUND(SUM(t.bill_total) - SUM(t.pay_total),2) as gross_profit FROM timesheets t JOIN clients c ON t.client_id = c.client_id WHERE t.approved = true GROUP BY c.company_name ORDER BY total_billed DESC LIMIT {limit}".into(),
|
||||
category: "analytics".into(),
|
||||
},
|
||||
ToolDef {
|
||||
name: "recruiter_performance".into(),
|
||||
description: "Show recruiter performance: placements, unique candidates, and total revenue generated.".into(),
|
||||
permission: Permission::Read,
|
||||
parameters: vec![
|
||||
ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Top N recruiters".into(), default: Some(serde_json::json!(10)) },
|
||||
],
|
||||
returns: "Recruiter name, placement count, unique candidates, total revenue".into(),
|
||||
sql_template: "SELECT p.recruiter, COUNT(DISTINCT p.placement_id) as placements, COUNT(DISTINCT p.candidate_id) as unique_candidates, ROUND(SUM(t.bill_total),2) as total_revenue FROM placements p JOIN timesheets t ON p.placement_id = t.placement_id GROUP BY p.recruiter ORDER BY total_revenue DESC LIMIT {limit}".into(),
|
||||
category: "analytics".into(),
|
||||
},
|
||||
ToolDef {
|
||||
name: "cold_leads".into(),
|
||||
description: "Find candidates who were called multiple times but never placed — potential lost opportunities.".into(),
|
||||
permission: Permission::Read,
|
||||
parameters: vec![
|
||||
ParamDef { name: "min_calls".into(), param_type: "integer".into(), required: false, description: "Minimum call count".into(), default: Some(serde_json::json!(5)) },
|
||||
ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
|
||||
],
|
||||
returns: "Candidates with call counts who were never placed".into(),
|
||||
sql_template: "SELECT c.candidate_id, c.first_name, c.last_name, c.phone, c.vertical, cl.calls FROM candidates c JOIN (SELECT candidate_id, COUNT(*) as calls FROM call_log GROUP BY candidate_id HAVING COUNT(*) >= {min_calls}) cl ON c.candidate_id = cl.candidate_id WHERE c.candidate_id NOT IN (SELECT DISTINCT candidate_id FROM placements) ORDER BY cl.calls DESC LIMIT {limit}".into(),
|
||||
category: "analytics".into(),
|
||||
},
|
||||
ToolDef {
|
||||
name: "open_jobs".into(),
|
||||
description: "List open job orders with client, title, rates, and location.".into(),
|
||||
permission: Permission::Read,
|
||||
parameters: vec![
|
||||
ParamDef { name: "vertical".into(), param_type: "string".into(), required: false, description: "Filter by vertical (IT, Healthcare, etc)".into(), default: None },
|
||||
ParamDef { name: "city".into(), param_type: "string".into(), required: false, description: "Filter by city".into(), default: None },
|
||||
ParamDef { name: "limit".into(), param_type: "integer".into(), required: false, description: "Max results".into(), default: Some(serde_json::json!(20)) },
|
||||
],
|
||||
returns: "Open job orders with details".into(),
|
||||
sql_template: "SELECT j.job_order_id, c.company_name, j.title, j.vertical, j.city, j.state, j.bill_rate, j.pay_rate, j.status FROM job_orders j JOIN clients c ON j.client_id = c.client_id WHERE j.status = 'open' {vertical_filter} {city_filter} ORDER BY j.bill_rate DESC LIMIT {limit}".into(),
|
||||
category: "jobs".into(),
|
||||
},
|
||||
];
|
||||
|
||||
let mut reg = self.tools.write().await;
|
||||
for tool in tools {
|
||||
reg.insert(tool.name.clone(), tool);
|
||||
}
|
||||
}
|
||||
|
||||
/// Get tool definition (for LLM tool-use schema).
|
||||
pub async fn get_tool(&self, name: &str) -> Option<ToolDef> {
|
||||
self.tools.read().await.get(name).cloned()
|
||||
}
|
||||
|
||||
/// List all tools (for LLM tool discovery).
|
||||
pub async fn list_tools(&self) -> Vec<ToolDef> {
|
||||
self.tools.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// Build SQL from tool parameters. Validates and sanitizes.
|
||||
pub fn build_sql(tool: &ToolDef, params: &serde_json::Value) -> Result<String, String> {
|
||||
let mut sql = tool.sql_template.clone();
|
||||
let params = params.as_object().ok_or("params must be an object")?;
|
||||
|
||||
// Replace named parameters
|
||||
for param_def in &tool.parameters {
|
||||
let value = params.get(¶m_def.name)
|
||||
.or(param_def.default.as_ref());
|
||||
|
||||
match param_def.name.as_str() {
|
||||
// Handle filter parameters (conditional WHERE clauses)
|
||||
"skills" => {
|
||||
if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
|
||||
let filters: Vec<String> = v.split(',')
|
||||
.map(|s| format!("skills LIKE '%{}%'", s.trim().replace('\'', "''")))
|
||||
.collect();
|
||||
sql = sql.replace("{skills_filter}", &format!("AND ({})", filters.join(" OR ")));
|
||||
} else {
|
||||
sql = sql.replace("{skills_filter}", "");
|
||||
}
|
||||
}
|
||||
"city" => {
|
||||
if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
|
||||
sql = sql.replace("{city_filter}", &format!("AND city = '{}'", v.replace('\'', "''")));
|
||||
} else {
|
||||
sql = sql.replace("{city_filter}", "");
|
||||
}
|
||||
}
|
||||
"state" => {
|
||||
if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
|
||||
sql = sql.replace("{state_filter}", &format!("AND state = '{}'", v.replace('\'', "''")));
|
||||
} else {
|
||||
sql = sql.replace("{state_filter}", "");
|
||||
}
|
||||
}
|
||||
"vertical" => {
|
||||
if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
|
||||
sql = sql.replace("{vertical_filter}", &format!("AND j.vertical = '{}'", v.replace('\'', "''")));
|
||||
} else {
|
||||
sql = sql.replace("{vertical_filter}", "");
|
||||
}
|
||||
}
|
||||
"status" => {
|
||||
if let Some(v) = value.and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
|
||||
sql = sql.replace("{status_filter}", &format!("AND status = '{}'", v.replace('\'', "''")));
|
||||
} else {
|
||||
sql = sql.replace("{status_filter}", "");
|
||||
}
|
||||
}
|
||||
"min_years" => {
|
||||
if let Some(v) = value.and_then(|v| v.as_i64()) {
|
||||
sql = sql.replace("{years_filter}", &format!("AND years_experience >= {v}"));
|
||||
} else {
|
||||
sql = sql.replace("{years_filter}", "");
|
||||
}
|
||||
}
|
||||
// Direct substitution for simple params
|
||||
_ => {
|
||||
let placeholder = format!("{{{}}}", param_def.name);
|
||||
if sql.contains(&placeholder) {
|
||||
let val_str = match value {
|
||||
Some(serde_json::Value::String(s)) => s.replace('\'', "''"),
|
||||
Some(serde_json::Value::Number(n)) => n.to_string(),
|
||||
Some(serde_json::Value::Bool(b)) => b.to_string(),
|
||||
Some(v) => v.to_string(),
|
||||
None if param_def.required => return Err(format!("missing required param: {}", param_def.name)),
|
||||
None => continue,
|
||||
};
|
||||
sql = sql.replace(&placeholder, &val_str);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(sql)
|
||||
}
|
||||
|
||||
/// Log a tool invocation.
|
||||
pub async fn log_invocation(&self, inv: ToolInvocation) {
|
||||
self.audit_log.write().await.push(inv);
|
||||
}
|
||||
|
||||
/// Get recent audit log.
|
||||
pub async fn recent_audit(&self, limit: usize) -> Vec<ToolInvocation> {
|
||||
let log = self.audit_log.read().await;
|
||||
let start = log.len().saturating_sub(limit);
|
||||
log[start..].iter().rev().cloned().collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::registry::{ToolInvocation, ToolRegistry};
|
||||
use crate::tools::ToolState;
|
||||
|
||||
pub fn router(state: ToolState) -> Router {
|
||||
Router::new()
|
||||
.route("/", get(list_tools))
|
||||
.route("/{name}", get(get_tool))
|
||||
.route("/{name}/call", post(call_tool))
|
||||
.route("/audit", get(audit_log))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
/// List all available tools (for LLM tool-use discovery).
|
||||
async fn list_tools(State(state): State<ToolState>) -> impl IntoResponse {
|
||||
let tools = state.registry.list_tools().await;
|
||||
// Return in MCP-compatible format
|
||||
let tool_list: Vec<serde_json::Value> = tools.iter().map(|t| {
|
||||
serde_json::json!({
|
||||
"name": t.name,
|
||||
"description": t.description,
|
||||
"permission": t.permission,
|
||||
"category": t.category,
|
||||
"parameters": t.parameters.iter().map(|p| {
|
||||
serde_json::json!({
|
||||
"name": p.name,
|
||||
"type": p.param_type,
|
||||
"required": p.required,
|
||||
"description": p.description,
|
||||
"default": p.default,
|
||||
})
|
||||
}).collect::<Vec<_>>(),
|
||||
})
|
||||
}).collect();
|
||||
Json(tool_list)
|
||||
}
|
||||
|
||||
/// Get a specific tool definition.
|
||||
async fn get_tool(
|
||||
State(state): State<ToolState>,
|
||||
Path(name): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match state.registry.get_tool(&name).await {
|
||||
Some(tool) => Ok(Json(tool)),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("tool not found: {name}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Call a tool — validate params, execute SQL, log invocation.
|
||||
#[derive(Deserialize)]
|
||||
struct CallRequest {
|
||||
params: serde_json::Value,
|
||||
agent: String,
|
||||
}
|
||||
|
||||
async fn call_tool(
|
||||
State(state): State<ToolState>,
|
||||
Path(name): Path<String>,
|
||||
Json(req): Json<CallRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let tool = match state.registry.get_tool(&name).await {
|
||||
Some(t) => t,
|
||||
None => return Err((StatusCode::NOT_FOUND, format!("tool not found: {name}"))),
|
||||
};
|
||||
|
||||
tracing::info!("tool call: {} by agent '{}' with {:?}", name, req.agent, req.params);
|
||||
|
||||
// Build SQL from params
|
||||
let sql = match ToolRegistry::build_sql(&tool, &req.params) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
state.registry.log_invocation(ToolInvocation {
|
||||
id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
|
||||
tool_name: name.clone(),
|
||||
agent: req.agent.clone(),
|
||||
params: req.params.clone(),
|
||||
permission: tool.permission.clone(),
|
||||
timestamp: chrono::Utc::now(),
|
||||
success: false,
|
||||
error: Some(e.clone()),
|
||||
rows_returned: None,
|
||||
}).await;
|
||||
return Err((StatusCode::BAD_REQUEST, e));
|
||||
}
|
||||
};
|
||||
|
||||
// Truth gate — same contract as queryd /sql (P42-002). Rejects
|
||||
// destructive verbs + empty SQL. Scrum iter 11 CF-1 + CF-2 on this
|
||||
// file: tools executed model-provided SQL parameters without any
|
||||
// validation. Close the gap here so the parallel surface has the
|
||||
// same safety floor as queryd.
|
||||
let ctx = serde_json::json!({ "sql": sql });
|
||||
for outcome in state.truth.evaluate("sql_query", &ctx) {
|
||||
if outcome.passed {
|
||||
if let truth::RuleAction::Reject { message } | truth::RuleAction::Block { message } = &outcome.action {
|
||||
tracing::warn!("tool {name}: SQL blocked by truth gate ({}): {message}", outcome.rule_id);
|
||||
state.registry.log_invocation(ToolInvocation {
|
||||
id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
|
||||
tool_name: name.clone(),
|
||||
agent: req.agent.clone(),
|
||||
params: req.params.clone(),
|
||||
permission: tool.permission.clone(),
|
||||
timestamp: chrono::Utc::now(),
|
||||
success: false,
|
||||
error: Some(format!("truth gate: {message}")),
|
||||
rows_returned: None,
|
||||
}).await;
|
||||
return Err((StatusCode::FORBIDDEN, message.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Execute via query engine
|
||||
let result = state.query_fn.execute(&sql).await;
|
||||
|
||||
match result {
|
||||
Ok((rows, row_count)) => {
|
||||
state.registry.log_invocation(ToolInvocation {
|
||||
id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
|
||||
tool_name: name.clone(),
|
||||
agent: req.agent,
|
||||
params: req.params,
|
||||
permission: tool.permission,
|
||||
timestamp: chrono::Utc::now(),
|
||||
success: true,
|
||||
error: None,
|
||||
rows_returned: Some(row_count),
|
||||
}).await;
|
||||
|
||||
Ok(Json(serde_json::json!({
|
||||
"tool": name,
|
||||
"rows": rows,
|
||||
"row_count": row_count,
|
||||
"sql": sql,
|
||||
})))
|
||||
}
|
||||
Err(e) => {
|
||||
state.registry.log_invocation(ToolInvocation {
|
||||
id: format!("inv-{}", chrono::Utc::now().timestamp_millis()),
|
||||
tool_name: name,
|
||||
agent: req.agent,
|
||||
params: req.params,
|
||||
permission: tool.permission,
|
||||
timestamp: chrono::Utc::now(),
|
||||
success: false,
|
||||
error: Some(e.clone()),
|
||||
rows_returned: None,
|
||||
}).await;
|
||||
|
||||
Err((StatusCode::INTERNAL_SERVER_ERROR, e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AuditQuery {
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
async fn audit_log(
|
||||
State(state): State<ToolState>,
|
||||
Query(q): Query<AuditQuery>,
|
||||
) -> impl IntoResponse {
|
||||
let log = state.registry.recent_audit(q.limit.unwrap_or(50)).await;
|
||||
Json(log)
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Claude (Anthropic) adapter.
|
||||
//!
|
||||
//! POST `https://api.anthropic.com/v1/messages`. Auth via `x-api-key`
|
||||
//! header (not bearer) + required `anthropic-version` header. Payload
|
||||
//! is NOT OpenAI-compatible — response text lives at
|
||||
//! `content[0].text`. Phase 40 deliverable. System prompts travel in
|
||||
//! a top-level `system` field, separate from the `messages` array.
|
||||
|
||||
use std::time::Duration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
||||
|
||||
const CLAUDE_BASE_URL: &str = "https://api.anthropic.com/v1";
|
||||
const CLAUDE_API_VERSION: &str = "2023-06-01";
|
||||
const CLAUDE_TIMEOUT_SECS: u64 = 180;
|
||||
|
||||
pub fn resolve_claude_key() -> Option<String> {
|
||||
if let Ok(k) = std::env::var("ANTHROPIC_API_KEY") {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
for path in ["/home/profit/.env", "/root/.env"] {
|
||||
if let Ok(raw) = std::fs::read_to_string(path) {
|
||||
for line in raw.lines() {
|
||||
if let Some(rest) = line.strip_prefix("ANTHROPIC_API_KEY=") {
|
||||
let k = rest.trim().trim_matches('"').trim_matches('\'');
|
||||
if !k.is_empty() { return Some(k.to_string()); }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn chat(
|
||||
key: &str,
|
||||
req: &ChatRequest,
|
||||
) -> Result<ChatResponse, String> {
|
||||
// Strip the "claude/" prefix if the caller used the namespaced form.
|
||||
let model = req.model.strip_prefix("claude/").unwrap_or(&req.model).to_string();
|
||||
|
||||
// Anthropic carries system prompts outside the messages array.
|
||||
// Concatenate any system-role messages into a single system string;
|
||||
// keep user + assistant messages in `messages`.
|
||||
let mut system_parts: Vec<String> = Vec::new();
|
||||
let mut msgs: Vec<AnMessage> = Vec::new();
|
||||
for m in &req.messages {
|
||||
if m.role == "system" {
|
||||
system_parts.push(m.content.clone());
|
||||
} else {
|
||||
// Anthropic expects strictly "user" or "assistant"; anything
|
||||
// else we normalize to "user".
|
||||
let role = if m.role == "assistant" { "assistant" } else { "user" };
|
||||
msgs.push(AnMessage { role: role.to_string(), content: m.content.clone() });
|
||||
}
|
||||
}
|
||||
let system = if system_parts.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(system_parts.join("\n\n"))
|
||||
};
|
||||
|
||||
let body = AnChatBody {
|
||||
model: model.clone(),
|
||||
messages: msgs,
|
||||
max_tokens: req.max_tokens.unwrap_or(800),
|
||||
temperature: req.temperature.unwrap_or(0.3),
|
||||
system,
|
||||
};
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(CLAUDE_TIMEOUT_SECS))
|
||||
.build()
|
||||
.map_err(|e| format!("build client: {e}"))?;
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let resp = client
|
||||
.post(format!("{}/messages", CLAUDE_BASE_URL))
|
||||
.header("x-api-key", key)
|
||||
.header("anthropic-version", CLAUDE_API_VERSION)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("api.anthropic.com unreachable: {e}"))?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_else(|_| "?".into());
|
||||
return Err(format!("claude {}: {}", status, body));
|
||||
}
|
||||
|
||||
let parsed: AnChatResponse = resp.json().await
|
||||
.map_err(|e| format!("invalid claude response: {e}"))?;
|
||||
|
||||
let latency_ms = t0.elapsed().as_millis();
|
||||
let text = parsed.content.into_iter()
|
||||
.find(|b| b.block_type == "text")
|
||||
.map(|b| b.text)
|
||||
.unwrap_or_default();
|
||||
|
||||
let prompt_tokens = parsed.usage.as_ref().map(|u| u.input_tokens).unwrap_or_else(|| {
|
||||
let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
|
||||
((chars + 3) / 4) as u32
|
||||
});
|
||||
let completion_tokens = parsed.usage.as_ref().map(|u| u.output_tokens).unwrap_or_else(|| {
|
||||
((text.chars().count() + 3) / 4) as u32
|
||||
});
|
||||
|
||||
tracing::info!(
|
||||
target: "v1.chat",
|
||||
provider = "claude",
|
||||
model = %model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
latency_ms = latency_ms as u64,
|
||||
"claude chat completed",
|
||||
);
|
||||
|
||||
Ok(ChatResponse {
|
||||
id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
|
||||
object: "chat.completion",
|
||||
created: chrono::Utc::now().timestamp(),
|
||||
model,
|
||||
choices: vec![Choice {
|
||||
index: 0,
|
||||
message: Message { role: "assistant".into(), content: text },
|
||||
finish_reason: parsed.stop_reason.unwrap_or_else(|| "stop".into()),
|
||||
}],
|
||||
usage: UsageBlock {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// -- Anthropic Messages API wire shapes --
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct AnChatBody {
|
||||
model: String,
|
||||
messages: Vec<AnMessage>,
|
||||
max_tokens: u32,
|
||||
temperature: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
system: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct AnMessage { role: String, content: String }
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AnChatResponse {
|
||||
content: Vec<AnContentBlock>,
|
||||
#[serde(default, rename = "stop_reason")]
|
||||
stop_reason: Option<String>,
|
||||
#[serde(default)]
|
||||
usage: Option<AnUsage>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AnContentBlock {
|
||||
#[serde(rename = "type")]
|
||||
block_type: String,
|
||||
#[serde(default)]
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AnUsage { input_tokens: u32, output_tokens: u32 }
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resolve_claude_key_does_not_panic() {
|
||||
let _ = resolve_claude_key();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_body_serializes_with_separate_system() {
|
||||
let body = AnChatBody {
|
||||
model: "claude-3-5-sonnet-latest".into(),
|
||||
messages: vec![
|
||||
AnMessage { role: "user".into(), content: "hi".into() },
|
||||
],
|
||||
max_tokens: 800,
|
||||
temperature: 0.3,
|
||||
system: Some("You are helpful.".into()),
|
||||
};
|
||||
let json = serde_json::to_string(&body).unwrap();
|
||||
assert!(json.contains("\"system\":\"You are helpful.\""));
|
||||
assert!(json.contains("\"messages\""));
|
||||
assert!(json.contains("\"max_tokens\":800"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_omits_system_when_none() {
|
||||
let body = AnChatBody {
|
||||
model: "claude-3-5-sonnet-latest".into(),
|
||||
messages: vec![AnMessage { role: "user".into(), content: "hi".into() }],
|
||||
max_tokens: 800,
|
||||
temperature: 0.3,
|
||||
system: None,
|
||||
};
|
||||
let json = serde_json::to_string(&body).unwrap();
|
||||
assert!(!json.contains("\"system\""), "system field should be skipped when None: {json}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_prefix_strip_preserves_bare_names() {
|
||||
let cases = [
|
||||
("claude/claude-3-5-sonnet-latest", "claude-3-5-sonnet-latest"),
|
||||
("claude-3-5-sonnet-latest", "claude-3-5-sonnet-latest"),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
let out = input.strip_prefix("claude/").unwrap_or(input);
|
||||
assert_eq!(out, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
//! Gemini adapter — Google's Generative Language API.
|
||||
//!
|
||||
//! POST `https://generativelanguage.googleapis.com/v1beta/models/
|
||||
//! {model}:generateContent?key=<API_KEY>`. Auth via query-string key
|
||||
//! (not bearer). Payload shape is NOT OpenAI-compatible — we map
|
||||
//! messages → contents + parts, extract response from `candidates[0]
|
||||
//! .content.parts[0].text`. Phase 40 deliverable; gate: `/v1/chat`
|
||||
//! with a prefixed or explicit gemini model returns normally.
|
||||
|
||||
use std::time::Duration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
||||
|
||||
const GEMINI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta";
|
||||
const GEMINI_TIMEOUT_SECS: u64 = 180;
|
||||
|
||||
pub fn resolve_gemini_key() -> Option<String> {
|
||||
if let Ok(k) = std::env::var("GEMINI_API_KEY") {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
for path in ["/home/profit/.env", "/root/.env"] {
|
||||
if let Ok(raw) = std::fs::read_to_string(path) {
|
||||
for line in raw.lines() {
|
||||
if let Some(rest) = line.strip_prefix("GEMINI_API_KEY=") {
|
||||
let k = rest.trim().trim_matches('"').trim_matches('\'');
|
||||
if !k.is_empty() { return Some(k.to_string()); }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn chat(
|
||||
key: &str,
|
||||
req: &ChatRequest,
|
||||
) -> Result<ChatResponse, String> {
|
||||
// Strip the "gemini/" prefix if the caller used the namespaced form.
|
||||
let model = req.model.strip_prefix("gemini/").unwrap_or(&req.model).to_string();
|
||||
|
||||
// Gemini splits system prompt from conversation differently.
|
||||
// Simplest working mapping: concatenate any system messages at the
|
||||
// top of a single user turn, then append user/assistant turns as
|
||||
// separate contents entries. Covers the common single-turn case
|
||||
// the scrum pipeline uses.
|
||||
let mut contents: Vec<GmContent> = Vec::new();
|
||||
for m in &req.messages {
|
||||
let role = match m.role.as_str() {
|
||||
"system" | "user" => "user",
|
||||
_ => "model",
|
||||
};
|
||||
contents.push(GmContent {
|
||||
role: role.to_string(),
|
||||
parts: vec![GmPart { text: m.content.clone() }],
|
||||
});
|
||||
}
|
||||
|
||||
let body = GmChatBody {
|
||||
contents,
|
||||
generation_config: GmGenerationConfig {
|
||||
temperature: req.temperature.unwrap_or(0.3),
|
||||
max_output_tokens: req.max_tokens.unwrap_or(800),
|
||||
},
|
||||
};
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(GEMINI_TIMEOUT_SECS))
|
||||
.build()
|
||||
.map_err(|e| format!("build client: {e}"))?;
|
||||
|
||||
let url = format!("{}/models/{}:generateContent?key={}", GEMINI_BASE_URL, model, key);
|
||||
let t0 = std::time::Instant::now();
|
||||
let resp = client
|
||||
.post(&url)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("generativelanguage.googleapis.com unreachable: {e}"))?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_else(|_| "?".into());
|
||||
return Err(format!("gemini {}: {}", status, body));
|
||||
}
|
||||
|
||||
let parsed: GmChatResponse = resp.json().await
|
||||
.map_err(|e| format!("invalid gemini response: {e}"))?;
|
||||
|
||||
let latency_ms = t0.elapsed().as_millis();
|
||||
let candidate = parsed.candidates.into_iter().next()
|
||||
.ok_or_else(|| "gemini returned no candidates".to_string())?;
|
||||
let text = candidate.content.parts.into_iter()
|
||||
.next()
|
||||
.map(|p| p.text)
|
||||
.unwrap_or_default();
|
||||
|
||||
let prompt_tokens = parsed.usage_metadata.as_ref()
|
||||
.map(|u| u.prompt_token_count)
|
||||
.unwrap_or_else(|| {
|
||||
let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
|
||||
((chars + 3) / 4) as u32
|
||||
});
|
||||
let completion_tokens = parsed.usage_metadata.as_ref()
|
||||
.map(|u| u.candidates_token_count)
|
||||
.unwrap_or_else(|| ((text.chars().count() + 3) / 4) as u32);
|
||||
|
||||
tracing::info!(
|
||||
target: "v1.chat",
|
||||
provider = "gemini",
|
||||
model = %model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
latency_ms = latency_ms as u64,
|
||||
"gemini chat completed",
|
||||
);
|
||||
|
||||
Ok(ChatResponse {
|
||||
id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
|
||||
object: "chat.completion",
|
||||
created: chrono::Utc::now().timestamp(),
|
||||
model,
|
||||
choices: vec![Choice {
|
||||
index: 0,
|
||||
message: Message { role: "assistant".into(), content: text },
|
||||
finish_reason: candidate.finish_reason.unwrap_or_else(|| "stop".into()),
|
||||
}],
|
||||
usage: UsageBlock {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// -- Gemini wire shapes --
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GmChatBody {
|
||||
contents: Vec<GmContent>,
|
||||
#[serde(rename = "generationConfig")]
|
||||
generation_config: GmGenerationConfig,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GmContent {
|
||||
role: String,
|
||||
parts: Vec<GmPart>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GmPart { text: String }
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GmGenerationConfig {
|
||||
temperature: f64,
|
||||
max_output_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GmChatResponse {
|
||||
candidates: Vec<GmCandidate>,
|
||||
#[serde(default, rename = "usageMetadata")]
|
||||
usage_metadata: Option<GmUsage>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GmCandidate {
|
||||
content: GmContentResp,
|
||||
#[serde(default, rename = "finishReason")]
|
||||
finish_reason: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GmContentResp { parts: Vec<GmPartResp> }
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GmPartResp { #[serde(default)] text: String }
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GmUsage {
|
||||
prompt_token_count: u32,
|
||||
candidates_token_count: u32,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resolve_gemini_key_does_not_panic() {
|
||||
let _ = resolve_gemini_key();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_body_serializes_to_gemini_shape() {
|
||||
let body = GmChatBody {
|
||||
contents: vec![
|
||||
GmContent {
|
||||
role: "user".into(),
|
||||
parts: vec![GmPart { text: "hello".into() }],
|
||||
},
|
||||
],
|
||||
generation_config: GmGenerationConfig {
|
||||
temperature: 0.3,
|
||||
max_output_tokens: 800,
|
||||
},
|
||||
};
|
||||
let json = serde_json::to_string(&body).unwrap();
|
||||
assert!(json.contains("\"contents\""));
|
||||
assert!(json.contains("\"parts\""));
|
||||
// camelCase per Gemini API
|
||||
assert!(json.contains("\"generationConfig\""));
|
||||
assert!(json.contains("\"maxOutputTokens\":800"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_prefix_strip_preserves_bare_names() {
|
||||
let cases = [
|
||||
("gemini/gemini-2.0-flash", "gemini-2.0-flash"),
|
||||
("gemini-2.0-flash", "gemini-2.0-flash"),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
let out = input.strip_prefix("gemini/").unwrap_or(input);
|
||||
assert_eq!(out, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
//! Phase 40 (early deliverable) — Langfuse tracing from the Rust gateway.
|
||||
//!
|
||||
//! Langfuse is already running at `http://localhost:3001` with the
|
||||
//! staffing project — `mcp-server/tracing.ts` has been emitting traces
|
||||
//! for months. This file bridges the gap: every `/v1/chat` call now
|
||||
//! appears in the same Langfuse project, alongside traces from the
|
||||
//! Bun side.
|
||||
//!
|
||||
//! Design:
|
||||
//! - Fire-and-forget — emit spawns a tokio task. Response to client
|
||||
//! never waits on Langfuse, never fails because of it. If Langfuse
|
||||
//! is down or slow, we silently drop traces (and `tracing::warn!`).
|
||||
//! - Two events per call — `trace-create` + `generation-create`. The
|
||||
//! generation nests under the trace so Langfuse UI shows the whole
|
||||
//! thing as one clickable row.
|
||||
//! - No SDK — plain HTTP POST to `/api/public/ingestion` with Basic
|
||||
//! auth. Keeps the Rust dependency graph small.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::Message;
|
||||
|
||||
const INGESTION_PATH: &str = "/api/public/ingestion";
|
||||
const EMIT_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LangfuseClient {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
base_url: String,
|
||||
public_key: String,
|
||||
secret_key: String,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl LangfuseClient {
|
||||
pub fn new(base_url: String, public_key: String, secret_key: String) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(EMIT_TIMEOUT_SECS))
|
||||
.build()
|
||||
.expect("langfuse client init");
|
||||
Self { inner: Arc::new(Inner { base_url, public_key, secret_key, http }) }
|
||||
}
|
||||
|
||||
/// Resolve a LangfuseClient from the same conventions the TS side
|
||||
/// already uses. Returns None if keys are missing — caller logs a
|
||||
/// warning and omits tracing (non-fatal).
|
||||
///
|
||||
/// Sources checked in order:
|
||||
/// 1. LANGFUSE_URL + LANGFUSE_PUBLIC_KEY + LANGFUSE_SECRET_KEY env
|
||||
/// 2. /root/llm_team_config.json (if it carries langfuse keys)
|
||||
/// 3. Hardcoded staffing-project defaults matching mcp-server
|
||||
/// (pk-lf-staffing / sk-lf-staffing-secret / localhost:3001)
|
||||
pub fn from_env_or_defaults() -> Option<Self> {
|
||||
let base = std::env::var("LANGFUSE_URL")
|
||||
.unwrap_or_else(|_| "http://localhost:3001".to_string());
|
||||
let pk = std::env::var("LANGFUSE_PUBLIC_KEY")
|
||||
.unwrap_or_else(|_| "pk-lf-staffing".to_string());
|
||||
let sk = std::env::var("LANGFUSE_SECRET_KEY")
|
||||
.unwrap_or_else(|_| "sk-lf-staffing-secret".to_string());
|
||||
if pk.trim().is_empty() || sk.trim().is_empty() { return None; }
|
||||
Some(Self::new(base, pk, sk))
|
||||
}
|
||||
|
||||
/// Fire-and-forget emit. Never blocks, never errors upward.
|
||||
pub fn emit_chat(&self, ev: ChatTrace) {
|
||||
let this = self.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = this.emit_chat_inner(ev).await {
|
||||
tracing::warn!(target: "v1.langfuse", "trace drop: {e}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn emit_chat_inner(&self, ev: ChatTrace) -> Result<(), String> {
|
||||
let trace_id = uuid_v7_like();
|
||||
let gen_id = uuid_v7_like();
|
||||
let trace_ts = ev.start_time.clone();
|
||||
|
||||
let batch = IngestionBatch {
|
||||
batch: vec![
|
||||
IngestionEvent {
|
||||
id: uuid_v7_like(),
|
||||
timestamp: trace_ts.clone(),
|
||||
kind: "trace-create",
|
||||
body: serde_json::json!({
|
||||
"id": trace_id,
|
||||
"name": format!("v1.chat:{}", ev.provider),
|
||||
"input": serde_json::json!({
|
||||
"model": ev.model,
|
||||
"messages": ev.input,
|
||||
}),
|
||||
"metadata": serde_json::json!({
|
||||
"provider": ev.provider,
|
||||
"think": ev.think,
|
||||
}),
|
||||
}),
|
||||
},
|
||||
IngestionEvent {
|
||||
id: uuid_v7_like(),
|
||||
timestamp: ev.end_time.clone(),
|
||||
kind: "generation-create",
|
||||
body: serde_json::json!({
|
||||
"id": gen_id,
|
||||
"traceId": trace_id,
|
||||
"name": "chat",
|
||||
"model": ev.model,
|
||||
"modelParameters": serde_json::json!({
|
||||
"temperature": ev.temperature,
|
||||
"max_tokens": ev.max_tokens,
|
||||
"think": ev.think,
|
||||
}),
|
||||
"input": ev.input,
|
||||
"output": ev.output,
|
||||
"usage": serde_json::json!({
|
||||
"input": ev.prompt_tokens,
|
||||
"output": ev.completion_tokens,
|
||||
"total": ev.prompt_tokens + ev.completion_tokens,
|
||||
"unit": "TOKENS",
|
||||
}),
|
||||
"startTime": ev.start_time,
|
||||
"endTime": ev.end_time,
|
||||
"metadata": serde_json::json!({
|
||||
"provider": ev.provider,
|
||||
"latency_ms": ev.latency_ms,
|
||||
}),
|
||||
}),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let url = format!("{}{}", self.inner.base_url.trim_end_matches('/'), INGESTION_PATH);
|
||||
let resp = self.inner.http
|
||||
.post(url)
|
||||
.basic_auth(&self.inner.public_key, Some(&self.inner.secret_key))
|
||||
.json(&batch)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("POST failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("{}: {}", resp.status(), resp.text().await.unwrap_or_default()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything the v1.chat handler collects for one completed call.
|
||||
pub struct ChatTrace {
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub input: Vec<Message>,
|
||||
pub output: String,
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub temperature: Option<f64>,
|
||||
pub max_tokens: Option<u32>,
|
||||
pub think: Option<bool>,
|
||||
pub start_time: String,
|
||||
pub end_time: String,
|
||||
pub latency_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct IngestionBatch {
|
||||
batch: Vec<IngestionEvent>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct IngestionEvent {
|
||||
id: String,
|
||||
timestamp: String,
|
||||
#[serde(rename = "type")]
|
||||
kind: &'static str,
|
||||
body: serde_json::Value,
|
||||
}
|
||||
|
||||
/// UUIDv7-ish identifier — not strictly UUIDv7 but time-ordered and
|
||||
/// unique enough for event deduplication. Avoids a uuid crate dep.
|
||||
fn uuid_v7_like() -> String {
|
||||
let ts = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0);
|
||||
let rand = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.subsec_nanos())
|
||||
.unwrap_or(0);
|
||||
format!("{:016x}-{:08x}", ts, rand)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn from_env_or_defaults_returns_some_when_defaults_present() {
|
||||
// Defaults are hardcoded non-empty — should always produce Some.
|
||||
let c = LangfuseClient::from_env_or_defaults();
|
||||
assert!(c.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uuid_v7_like_produces_distinct_ids() {
|
||||
let a = uuid_v7_like();
|
||||
// Different nanosecond timestamps — but they might collide on
|
||||
// back-to-back calls. Loop a bit to ensure uniqueness surface.
|
||||
let b = uuid_v7_like();
|
||||
let c = uuid_v7_like();
|
||||
assert!(a != b || b != c, "three consecutive ids should have at least one distinct pair");
|
||||
assert!(a.contains('-'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingestion_batch_serializes_with_rename() {
|
||||
let ev = IngestionEvent {
|
||||
id: "e1".into(),
|
||||
timestamp: "2026-04-22T00:00:00Z".into(),
|
||||
kind: "trace-create",
|
||||
body: serde_json::json!({"name": "t"}),
|
||||
};
|
||||
let json = serde_json::to_string(&IngestionBatch { batch: vec![ev] }).unwrap();
|
||||
assert!(json.contains("\"type\":\"trace-create\""));
|
||||
assert!(json.contains("\"batch\":["));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
//! Phase 38 — Universal API skeleton (`/v1/*`).
|
||||
//!
|
||||
//! OpenAI-compatible shape on top of the existing aibridge → Ollama
|
||||
//! path. This is the thin slice: single provider, stateless, no
|
||||
//! streaming. Phase 39 replaces the direct Ollama call with a
|
||||
//! `ProviderAdapter` trait dispatch; Phase 40 adds routing + fallback.
|
||||
//!
|
||||
//! The shape matches OpenAI's `/v1/chat/completions` closely enough
|
||||
//! that clients using openai-compatible SDKs can point at us with the
|
||||
//! URL swap alone. We keep the endpoint path `/v1/chat` (not
|
||||
//! `/v1/chat/completions`) because our PRD declares the terser form;
|
||||
//! adding the alias is one line in Phase 39 when it matters.
|
||||
|
||||
pub mod ollama;
|
||||
pub mod ollama_cloud;
|
||||
pub mod openrouter;
|
||||
pub mod gemini;
|
||||
pub mod claude;
|
||||
pub mod langfuse_trace;
|
||||
pub mod respond;
|
||||
pub mod truth;
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
extract::State,
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
Json,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct V1State {
|
||||
pub ai_client: aibridge::client::AiClient,
|
||||
pub usage: Arc<RwLock<Usage>>,
|
||||
/// Ollama Cloud bearer token. Loaded at startup via
|
||||
/// `ollama_cloud::resolve_cloud_key()`. None = cloud routes 503.
|
||||
pub ollama_cloud_key: Option<String>,
|
||||
/// OpenRouter bearer token — free-tier rescue rung. Loaded at
|
||||
/// startup via `openrouter::resolve_openrouter_key()`. None means
|
||||
/// provider="openrouter" calls 503 rather than attempt. Same key
|
||||
/// sourcing as LLM Team UI so the two share one API quota.
|
||||
pub openrouter_key: Option<String>,
|
||||
/// Gemini API key (Google Generative Language). Loaded at startup
|
||||
/// via `gemini::resolve_gemini_key()`. None = provider="gemini"
|
||||
/// calls 503. Phase 40 deliverable.
|
||||
pub gemini_key: Option<String>,
|
||||
/// Anthropic Claude API key. Loaded at startup via
|
||||
/// `claude::resolve_claude_key()`. None = provider="claude" calls
|
||||
/// 503. Phase 40 deliverable.
|
||||
pub claude_key: Option<String>,
|
||||
/// Phase 40 early deliverable — Langfuse client. None = tracing
|
||||
/// disabled (keys missing or container unreachable). Traces are
|
||||
/// fire-and-forget: never block the response path.
|
||||
pub langfuse: Option<langfuse_trace::LangfuseClient>,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Serialize)]
|
||||
pub struct Usage {
|
||||
pub requests: u64,
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
#[serde(default)]
|
||||
pub by_provider: std::collections::HashMap<String, ProviderUsage>,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Serialize)]
|
||||
pub struct ProviderUsage {
|
||||
pub requests: u64,
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
pub fn router(state: V1State) -> Router {
|
||||
Router::new()
|
||||
.route("/chat", post(chat))
|
||||
.route("/respond", post(respond::respond))
|
||||
.route("/usage", get(usage))
|
||||
.route("/sessions", get(sessions))
|
||||
.route("/context", get(truth::context))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
// -- Shared types (OpenAI-compatible) --
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Message {
|
||||
pub role: String,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ChatRequest {
|
||||
pub model: String,
|
||||
pub messages: Vec<Message>,
|
||||
#[serde(default)]
|
||||
pub temperature: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub max_tokens: Option<u32>,
|
||||
/// Accepted for shape-compat but ignored in the thin slice —
|
||||
/// Phase 38 returns non-streaming even when the client asked for it.
|
||||
/// Phase 39+ wires real streaming.
|
||||
#[serde(default)]
|
||||
pub stream: Option<bool>,
|
||||
/// Non-OpenAI extension. Passes through to the provider's thinking
|
||||
/// toggle. Default: **false** — hot-path discipline for thinking
|
||||
/// models (qwen3.5, qwen3, gpt-oss) that otherwise burn the token
|
||||
/// budget on hidden reasoning before visible output starts,
|
||||
/// producing empty responses. Set true explicitly when calling an
|
||||
/// overseer / reasoning-heavy path.
|
||||
#[serde(default)]
|
||||
pub think: Option<bool>,
|
||||
/// Non-OpenAI extension. Selects the provider adapter. Accepted:
|
||||
/// - None / "ollama" / "local" → local Ollama via sidecar (default)
|
||||
/// - "ollama_cloud" / "cloud" → Ollama Cloud direct HTTPS
|
||||
/// Phase 40 adds a routing engine that picks this automatically
|
||||
/// from the model name; Phase 38/39 requires explicit selection.
|
||||
#[serde(default)]
|
||||
pub provider: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ChatResponse {
|
||||
pub id: String,
|
||||
pub object: &'static str,
|
||||
pub created: i64,
|
||||
pub model: String,
|
||||
pub choices: Vec<Choice>,
|
||||
pub usage: UsageBlock,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct Choice {
|
||||
pub index: u32,
|
||||
pub message: Message,
|
||||
pub finish_reason: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct UsageBlock {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
}
|
||||
|
||||
// -- Handlers --
|
||||
|
||||
/// Phase 39: resolve (provider, effective_model) from a ChatRequest.
|
||||
///
|
||||
/// Explicit `req.provider` wins. If absent, infer from a model-name
|
||||
/// prefix: "openrouter/..." → openrouter (strip prefix), "cloud/..." →
|
||||
/// ollama_cloud (strip prefix). Bare names default to "ollama".
|
||||
///
|
||||
/// The stripped model is what the upstream adapter expects:
|
||||
/// OpenRouter's API wants "openai/gpt-4o-mini", not
|
||||
/// "openrouter/openai/gpt-4o-mini".
|
||||
fn resolve_provider(req: &ChatRequest) -> (String, String) {
|
||||
if let Some(p) = req.provider.as_deref() {
|
||||
return (p.to_ascii_lowercase(), req.model.clone());
|
||||
}
|
||||
if let Some(rest) = req.model.strip_prefix("openrouter/") {
|
||||
return ("openrouter".to_string(), rest.to_string());
|
||||
}
|
||||
if let Some(rest) = req.model.strip_prefix("cloud/") {
|
||||
return ("ollama_cloud".to_string(), rest.to_string());
|
||||
}
|
||||
if let Some(rest) = req.model.strip_prefix("gemini/") {
|
||||
return ("gemini".to_string(), rest.to_string());
|
||||
}
|
||||
if let Some(rest) = req.model.strip_prefix("claude/") {
|
||||
return ("claude".to_string(), rest.to_string());
|
||||
}
|
||||
("ollama".to_string(), req.model.clone())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod resolve_provider_tests {
|
||||
use super::*;
|
||||
|
||||
fn mk_req(provider: Option<&str>, model: &str) -> ChatRequest {
|
||||
ChatRequest {
|
||||
model: model.to_string(),
|
||||
messages: vec![],
|
||||
temperature: None,
|
||||
max_tokens: None,
|
||||
stream: None,
|
||||
think: None,
|
||||
provider: provider.map(|s| s.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_provider_wins() {
|
||||
let r = mk_req(Some("openrouter"), "qwen3.5:latest");
|
||||
assert_eq!(resolve_provider(&r), ("openrouter".into(), "qwen3.5:latest".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_model_defaults_to_ollama() {
|
||||
let r = mk_req(None, "qwen3.5:latest");
|
||||
assert_eq!(resolve_provider(&r), ("ollama".into(), "qwen3.5:latest".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openrouter_prefix_infers_and_strips() {
|
||||
let r = mk_req(None, "openrouter/openai/gpt-4o-mini");
|
||||
assert_eq!(resolve_provider(&r), ("openrouter".into(), "openai/gpt-4o-mini".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloud_prefix_infers_and_strips() {
|
||||
let r = mk_req(None, "cloud/kimi-k2:1t");
|
||||
assert_eq!(resolve_provider(&r), ("ollama_cloud".into(), "kimi-k2:1t".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_provider_preserves_full_model_even_with_prefix() {
|
||||
// If caller provides both provider and a model with a prefix,
|
||||
// trust them — don't strip. The adapter will get the full model
|
||||
// string as-is.
|
||||
let r = mk_req(Some("openrouter"), "openrouter/openai/gpt-4o-mini");
|
||||
assert_eq!(resolve_provider(&r), ("openrouter".into(), "openrouter/openai/gpt-4o-mini".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_prefix_infers_and_strips() {
|
||||
let r = mk_req(None, "gemini/gemini-2.0-flash");
|
||||
assert_eq!(resolve_provider(&r), ("gemini".into(), "gemini-2.0-flash".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claude_prefix_infers_and_strips() {
|
||||
let r = mk_req(None, "claude/claude-3-5-sonnet-latest");
|
||||
assert_eq!(resolve_provider(&r), ("claude".into(), "claude-3-5-sonnet-latest".into()));
|
||||
}
|
||||
}
|
||||
|
||||
async fn chat(
|
||||
State(state): State<V1State>,
|
||||
Json(req): Json<ChatRequest>,
|
||||
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
|
||||
if req.messages.is_empty() {
|
||||
return Err((StatusCode::BAD_REQUEST, "messages must be non-empty".into()));
|
||||
}
|
||||
if req.stream.unwrap_or(false) {
|
||||
tracing::warn!("/v1/chat: stream=true requested but Phase 38 returns non-streaming");
|
||||
}
|
||||
|
||||
// Provider resolution: explicit `req.provider` wins; otherwise
|
||||
// infer from a model-name prefix. Phase 39 PRD gate example:
|
||||
// `model: "openrouter/openai/gpt-4o-mini"` → provider "openrouter",
|
||||
// adapter gets the stripped "openai/gpt-4o-mini".
|
||||
let (provider, effective_model) = resolve_provider(&req);
|
||||
let start_time = chrono::Utc::now();
|
||||
let start_instant = std::time::Instant::now();
|
||||
|
||||
// If we stripped a prefix, clone req with the effective model so
|
||||
// the adapter sees what the upstream provider expects (OpenRouter
|
||||
// wants "openai/gpt-4o-mini", not "openrouter/openai/gpt-4o-mini").
|
||||
let req_for_adapter: std::borrow::Cow<'_, ChatRequest> =
|
||||
if effective_model == req.model {
|
||||
std::borrow::Cow::Borrowed(&req)
|
||||
} else {
|
||||
let mut cloned = req.clone();
|
||||
cloned.model = effective_model.clone();
|
||||
std::borrow::Cow::Owned(cloned)
|
||||
};
|
||||
|
||||
let (resp, used_provider) = match provider.as_str() {
|
||||
"ollama" | "local" | "" => {
|
||||
let r = ollama::chat(&state.ai_client, &*req_for_adapter)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("ollama local: {e}")))?;
|
||||
(r, "ollama".to_string())
|
||||
}
|
||||
"ollama_cloud" | "cloud" => {
|
||||
let key = state.ollama_cloud_key.as_deref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"OLLAMA_CLOUD_KEY not configured".to_string(),
|
||||
))?;
|
||||
let r = ollama_cloud::chat(key, &*req_for_adapter)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("ollama cloud: {e}")))?;
|
||||
(r, "ollama_cloud".to_string())
|
||||
}
|
||||
"openrouter" | "openrouter_free" => {
|
||||
// Free-tier rescue rung. Added 2026-04-24 after iter 5
|
||||
// repeated Ollama Cloud 502s on kimi-k2:1t — OpenRouter
|
||||
// gives a different provider backbone as fallback.
|
||||
let key = state.openrouter_key.as_deref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"OPENROUTER_API_KEY not configured".to_string(),
|
||||
))?;
|
||||
let r = openrouter::chat(key, &*req_for_adapter)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("openrouter: {e}")))?;
|
||||
(r, "openrouter".to_string())
|
||||
}
|
||||
"gemini" => {
|
||||
// Phase 40 provider adapter. Google Generative Language
|
||||
// API via query-string key auth (not bearer).
|
||||
let key = state.gemini_key.as_deref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"GEMINI_API_KEY not configured".to_string(),
|
||||
))?;
|
||||
let r = gemini::chat(key, &*req_for_adapter)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("gemini: {e}")))?;
|
||||
(r, "gemini".to_string())
|
||||
}
|
||||
"claude" | "anthropic" => {
|
||||
// Phase 40 provider adapter. Anthropic Messages API via
|
||||
// x-api-key header + anthropic-version:2023-06-01.
|
||||
let key = state.claude_key.as_deref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"ANTHROPIC_API_KEY not configured".to_string(),
|
||||
))?;
|
||||
let r = claude::chat(key, &*req_for_adapter)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("claude: {e}")))?;
|
||||
(r, "claude".to_string())
|
||||
}
|
||||
other => {
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("unknown provider '{other}' — supported: ollama, ollama_cloud, openrouter, gemini, claude"),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
let end_time = chrono::Utc::now();
|
||||
let latency_ms = start_instant.elapsed().as_millis() as u64;
|
||||
|
||||
// Phase 40 — emit Langfuse trace. Fire-and-forget: the clone is
|
||||
// cheap (Arc inside), the tokio::spawn never blocks us, a dead
|
||||
// Langfuse just logs a warn. Client-visible response latency is
|
||||
// untouched.
|
||||
if let Some(lf) = &state.langfuse {
|
||||
let output = resp.choices.first()
|
||||
.map(|c| c.message.content.clone())
|
||||
.unwrap_or_default();
|
||||
lf.emit_chat(langfuse_trace::ChatTrace {
|
||||
provider: used_provider.clone(),
|
||||
model: resp.model.clone(),
|
||||
input: req.messages.clone(),
|
||||
output,
|
||||
prompt_tokens: resp.usage.prompt_tokens,
|
||||
completion_tokens: resp.usage.completion_tokens,
|
||||
temperature: req.temperature,
|
||||
max_tokens: req.max_tokens,
|
||||
think: req.think,
|
||||
start_time: start_time.to_rfc3339(),
|
||||
end_time: end_time.to_rfc3339(),
|
||||
latency_ms,
|
||||
});
|
||||
}
|
||||
|
||||
// Phase 40: per-provider usage tracking
|
||||
{
|
||||
let mut u = state.usage.write().await;
|
||||
u.requests += 1;
|
||||
u.prompt_tokens += resp.usage.prompt_tokens as u64;
|
||||
u.completion_tokens += resp.usage.completion_tokens as u64;
|
||||
u.total_tokens += resp.usage.total_tokens as u64;
|
||||
|
||||
let provider_usage = u.by_provider.entry(used_provider).or_default();
|
||||
provider_usage.requests += 1;
|
||||
provider_usage.prompt_tokens += resp.usage.prompt_tokens as u64;
|
||||
provider_usage.completion_tokens += resp.usage.completion_tokens as u64;
|
||||
provider_usage.total_tokens += resp.usage.total_tokens as u64;
|
||||
}
|
||||
|
||||
Ok(Json(resp))
|
||||
}
|
||||
|
||||
async fn usage(State(state): State<V1State>) -> impl IntoResponse {
|
||||
let snapshot = state.usage.read().await.clone();
|
||||
Json(snapshot)
|
||||
}
|
||||
|
||||
// Phase 38 is stateless — no session persistence yet. Return an empty
|
||||
// list in OpenAI-ish shape so clients that probe this endpoint don't
|
||||
// 404. Real session state lands in Phase 41 with the profile-system
|
||||
// expansion.
|
||||
async fn sessions() -> impl IntoResponse {
|
||||
Json(serde_json::json!({
|
||||
"data": [],
|
||||
"object": "list",
|
||||
"note": "Phase 38: stateless. Session state lands in Phase 41.",
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn chat_request_parses_openai_shape() {
|
||||
let raw = r#"{
|
||||
"model": "qwen3.5:latest",
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "Hi"}
|
||||
],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 100
|
||||
}"#;
|
||||
let r: ChatRequest = serde_json::from_str(raw).unwrap();
|
||||
assert_eq!(r.model, "qwen3.5:latest");
|
||||
assert_eq!(r.messages.len(), 2);
|
||||
assert_eq!(r.messages[0].role, "system");
|
||||
assert_eq!(r.messages[1].content, "Hi");
|
||||
assert_eq!(r.temperature, Some(0.2));
|
||||
assert_eq!(r.max_tokens, Some(100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_request_accepts_minimal() {
|
||||
let raw = r#"{
|
||||
"model": "any",
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}"#;
|
||||
let r: ChatRequest = serde_json::from_str(raw).unwrap();
|
||||
assert_eq!(r.temperature, None);
|
||||
assert_eq!(r.max_tokens, None);
|
||||
assert_eq!(r.stream, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_counter_default_is_zero() {
|
||||
let u = Usage::default();
|
||||
assert_eq!(u.requests, 0);
|
||||
assert_eq!(u.total_tokens, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
//! Phase 38 — Ollama shape adapter.
|
||||
//!
|
||||
//! Translates `/v1/chat` (OpenAI-compatible) requests into the
|
||||
//! existing aibridge `GenerateRequest` shape, and the `GenerateResponse`
|
||||
//! back into an OpenAI-compatible `ChatResponse`. This is a bridge,
|
||||
//! not a new client — aibridge + the Python sidecar stay as-is.
|
||||
//!
|
||||
//! Phase 39 replaces this direct call with a `ProviderAdapter` trait
|
||||
//! dispatch so the same `/v1/chat` handler routes to any provider.
|
||||
|
||||
use aibridge::client::{AiClient, GenerateRequest};
|
||||
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
||||
|
||||
pub async fn chat(client: &AiClient, req: &ChatRequest) -> Result<ChatResponse, String> {
|
||||
let (system, prompt) = flatten_messages(&req.messages);
|
||||
|
||||
let gen_req = GenerateRequest {
|
||||
prompt,
|
||||
model: Some(req.model.clone()),
|
||||
system: if system.is_empty() { None } else { Some(system) },
|
||||
temperature: req.temperature,
|
||||
max_tokens: req.max_tokens,
|
||||
// Phase 38 default: think=false. Hot-path discipline for
|
||||
// thinking models (qwen3.5, qwen3, gpt-oss) which otherwise
|
||||
// burn the max_tokens budget on hidden reasoning before any
|
||||
// visible output, producing empty responses. Callers that
|
||||
// actually want reasoning (overseers, T3+ tiers) opt in via
|
||||
// the `think: true` extension field. Phase 40 routing engine
|
||||
// flips this per task class.
|
||||
think: Some(req.think.unwrap_or(false)),
|
||||
};
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let resp = client.generate(gen_req).await.map_err(|e| e.to_string())?;
|
||||
let latency_ms = t0.elapsed().as_millis();
|
||||
|
||||
// Prefer sidecar-reported token counts when present. Fall back to
|
||||
// chars/4 estimate (biased safe ~15%, matches Phase 21 convention
|
||||
// in crates/aibridge/src/context.rs::estimate_tokens).
|
||||
let prompt_tokens = resp.tokens_evaluated
|
||||
.map(|n| n as u32)
|
||||
.unwrap_or_else(|| estimate_prompt_tokens(&req.messages));
|
||||
let completion_tokens = resp.tokens_generated
|
||||
.map(|n| n as u32)
|
||||
.unwrap_or_else(|| ((resp.text.chars().count() + 3) / 4) as u32);
|
||||
|
||||
tracing::info!(
|
||||
target: "v1.chat",
|
||||
model = %req.model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
latency_ms = latency_ms as u64,
|
||||
"ollama chat completed",
|
||||
);
|
||||
|
||||
Ok(ChatResponse {
|
||||
id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
|
||||
object: "chat.completion",
|
||||
created: chrono::Utc::now().timestamp(),
|
||||
model: resp.model,
|
||||
choices: vec![Choice {
|
||||
index: 0,
|
||||
message: Message {
|
||||
role: "assistant".into(),
|
||||
content: resp.text,
|
||||
},
|
||||
finish_reason: "stop".into(),
|
||||
}],
|
||||
usage: UsageBlock {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Public re-export of the flattener so sibling adapters (Ollama Cloud,
|
||||
/// future OpenRouter) can reuse the same shape collapse without
|
||||
/// duplicating the logic. Keeps `(system, prompt)` format consistent
|
||||
/// across providers.
|
||||
pub fn flatten_messages_public(messages: &[Message]) -> (String, String) {
|
||||
flatten_messages(messages)
|
||||
}
|
||||
|
||||
/// Collapse a message array into (system, prompt). Multiple system
|
||||
/// messages concatenate with a newline — matches OpenAI's documented
|
||||
/// behavior. Non-system messages become role-labeled turns.
|
||||
fn flatten_messages(messages: &[Message]) -> (String, String) {
|
||||
let mut system = String::new();
|
||||
let mut prompt = String::new();
|
||||
for m in messages {
|
||||
if m.role == "system" {
|
||||
if !system.is_empty() { system.push('\n'); }
|
||||
system.push_str(&m.content);
|
||||
} else {
|
||||
prompt.push_str(&m.role);
|
||||
prompt.push_str(": ");
|
||||
prompt.push_str(&m.content);
|
||||
prompt.push_str("\n\n");
|
||||
}
|
||||
}
|
||||
prompt.push_str("assistant:");
|
||||
(system, prompt)
|
||||
}
|
||||
|
||||
fn estimate_prompt_tokens(messages: &[Message]) -> u32 {
|
||||
let chars: usize = messages.iter().map(|m| m.content.chars().count()).sum();
|
||||
((chars + 3) / 4) as u32
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn flatten_separates_system_from_turns() {
|
||||
let msgs = vec![
|
||||
Message { role: "system".into(), content: "Rules here.".into() },
|
||||
Message { role: "user".into(), content: "Q1".into() },
|
||||
Message { role: "assistant".into(), content: "A1".into() },
|
||||
Message { role: "user".into(), content: "Q2".into() },
|
||||
];
|
||||
let (system, prompt) = flatten_messages(&msgs);
|
||||
assert_eq!(system, "Rules here.");
|
||||
assert!(prompt.contains("user: Q1"));
|
||||
assert!(prompt.contains("assistant: A1"));
|
||||
assert!(prompt.contains("user: Q2"));
|
||||
assert!(prompt.trim_end().ends_with("assistant:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flatten_concatenates_multiple_system_messages() {
|
||||
let msgs = vec![
|
||||
Message { role: "system".into(), content: "First.".into() },
|
||||
Message { role: "system".into(), content: "Second.".into() },
|
||||
Message { role: "user".into(), content: "Hi".into() },
|
||||
];
|
||||
let (system, _) = flatten_messages(&msgs);
|
||||
assert_eq!(system, "First.\nSecond.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flatten_with_no_system_returns_empty_system() {
|
||||
let msgs = vec![Message { role: "user".into(), content: "hi".into() }];
|
||||
let (system, prompt) = flatten_messages(&msgs);
|
||||
assert!(system.is_empty());
|
||||
assert!(prompt.contains("user: hi"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_tokens_chars_div_4_ceiling() {
|
||||
let msgs = vec![Message { role: "user".into(), content: "abcdefgh".into() }];
|
||||
// 8 chars / 4 = 2, with ceiling → 2
|
||||
assert_eq!(estimate_prompt_tokens(&msgs), 2);
|
||||
|
||||
let msgs2 = vec![Message { role: "user".into(), content: "abcdefghi".into() }];
|
||||
// 9 chars → (9+3)/4 = 3 (ceiling)
|
||||
assert_eq!(estimate_prompt_tokens(&msgs2), 3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
//! Phase 39 (first slice) — Ollama Cloud adapter.
|
||||
//!
|
||||
//! Direct HTTPS call to `https://ollama.com/api/generate` with Bearer
|
||||
//! auth. Mirrors the `tests/multi-agent/agent.ts::generateCloud` pattern
|
||||
//! exactly (same endpoint, same body shape, same Bearer header), so
|
||||
//! cloud calls from Rust behave identically to the TS hot path.
|
||||
//!
|
||||
//! Key sourcing priority:
|
||||
//! 1. Env var `OLLAMA_CLOUD_KEY` (matches agent.ts convention)
|
||||
//! 2. `/root/llm_team_config.json` → providers.ollama_cloud.api_key
|
||||
//! 3. Env var `OLLAMA_CLOUD_API_KEY` (LLM Team UI convention)
|
||||
//!
|
||||
//! First hit wins. Key is loaded once at gateway startup by
|
||||
//! `resolve_cloud_key()` and stored on `V1State.ollama_cloud_key`.
|
||||
|
||||
use std::time::Duration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
||||
|
||||
const CLOUD_BASE_URL: &str = "https://ollama.com";
|
||||
const CLOUD_TIMEOUT_SECS: u64 = 180;
|
||||
|
||||
/// Read the Ollama Cloud key from the three sanctioned sources. Returns
|
||||
/// None if none is set — callers must 503 rather than attempt a call.
|
||||
pub fn resolve_cloud_key() -> Option<String> {
|
||||
if let Ok(k) = std::env::var("OLLAMA_CLOUD_KEY") {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
if let Ok(raw) = std::fs::read_to_string("/root/llm_team_config.json") {
|
||||
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) {
|
||||
if let Some(k) = v.pointer("/providers/ollama_cloud/api_key").and_then(|x| x.as_str()) {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(k) = std::env::var("OLLAMA_CLOUD_API_KEY") {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn chat(
|
||||
key: &str,
|
||||
req: &ChatRequest,
|
||||
) -> Result<ChatResponse, String> {
|
||||
let (system, prompt) = super::ollama::flatten_messages_public(&req.messages);
|
||||
|
||||
let body = CloudGenerateBody {
|
||||
model: req.model.clone(),
|
||||
prompt,
|
||||
system: if system.is_empty() { None } else { Some(system) },
|
||||
stream: false,
|
||||
think: Some(req.think.unwrap_or(false)),
|
||||
options: CloudOptions {
|
||||
// Thinking cloud models need headroom — floor 400 to give
|
||||
// qwen3.5:397b / gpt-oss:120b reasoning room. Matches
|
||||
// agent.ts `Math.max(opts.max_tokens ?? 800, 400)` policy.
|
||||
num_predict: std::cmp::max(req.max_tokens.unwrap_or(800), 400),
|
||||
temperature: req.temperature.unwrap_or(0.3),
|
||||
},
|
||||
};
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(CLOUD_TIMEOUT_SECS))
|
||||
.build()
|
||||
.map_err(|e| format!("build client: {e}"))?;
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let resp = client
|
||||
.post(format!("{}/api/generate", CLOUD_BASE_URL))
|
||||
.bearer_auth(key)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("ollama.com unreachable: {e}"))?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_else(|_| "?".into());
|
||||
return Err(format!("ollama.com {}: {}", status, body));
|
||||
}
|
||||
|
||||
let parsed: CloudGenerateResponse = resp.json().await
|
||||
.map_err(|e| format!("invalid cloud response: {e}"))?;
|
||||
|
||||
let latency_ms = t0.elapsed().as_millis();
|
||||
let text = parsed.response.unwrap_or_default();
|
||||
|
||||
let prompt_tokens = parsed.prompt_eval_count.unwrap_or_else(|| {
|
||||
let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
|
||||
((chars + 3) / 4) as u32
|
||||
});
|
||||
let completion_tokens = parsed.eval_count.unwrap_or_else(|| {
|
||||
((text.chars().count() + 3) / 4) as u32
|
||||
});
|
||||
|
||||
tracing::info!(
|
||||
target: "v1.chat",
|
||||
provider = "ollama_cloud",
|
||||
model = %req.model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
latency_ms = latency_ms as u64,
|
||||
"cloud chat completed",
|
||||
);
|
||||
|
||||
Ok(ChatResponse {
|
||||
id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
|
||||
object: "chat.completion",
|
||||
created: chrono::Utc::now().timestamp(),
|
||||
model: parsed.model.unwrap_or_else(|| req.model.clone()),
|
||||
choices: vec![Choice {
|
||||
index: 0,
|
||||
message: Message { role: "assistant".into(), content: text },
|
||||
finish_reason: "stop".into(),
|
||||
}],
|
||||
usage: UsageBlock {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// -- Ollama /api/generate wire shapes --
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CloudGenerateBody {
|
||||
model: String,
|
||||
prompt: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
system: Option<String>,
|
||||
stream: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
think: Option<bool>,
|
||||
options: CloudOptions,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CloudOptions {
|
||||
num_predict: u32,
|
||||
temperature: f64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloudGenerateResponse {
|
||||
#[serde(default)]
|
||||
response: Option<String>,
|
||||
#[serde(default)]
|
||||
model: Option<String>,
|
||||
#[serde(default)]
|
||||
prompt_eval_count: Option<u32>,
|
||||
#[serde(default)]
|
||||
eval_count: Option<u32>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resolve_cloud_key_returns_none_when_no_sources_set() {
|
||||
// Only check shape — we can't reliably unset env vars in a test
|
||||
// that runs alongside others, and the file path is on disk.
|
||||
// If all three sources are empty the function returns None; if
|
||||
// any are set we expect Some. This just smoke-tests the call
|
||||
// doesn't panic.
|
||||
let _ = resolve_cloud_key();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloud_body_serializes_compact() {
|
||||
let body = CloudGenerateBody {
|
||||
model: "gpt-oss:120b".into(),
|
||||
prompt: "user: hi\n\nassistant:".into(),
|
||||
system: Some("Be terse.".into()),
|
||||
stream: false,
|
||||
think: Some(false),
|
||||
options: CloudOptions { num_predict: 400, temperature: 0.3 },
|
||||
};
|
||||
let json = serde_json::to_string(&body).unwrap();
|
||||
assert!(json.contains("\"model\":\"gpt-oss:120b\""));
|
||||
assert!(json.contains("\"stream\":false"));
|
||||
assert!(json.contains("\"num_predict\":400"));
|
||||
assert!(json.contains("\"think\":false"));
|
||||
assert!(json.contains("\"system\":\"Be terse.\""));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
//! OpenRouter adapter — free-tier rescue rung for /v1/chat.
|
||||
//!
|
||||
//! Direct HTTPS call to `https://openrouter.ai/api/v1/chat/completions`
|
||||
//! with Bearer auth. Mirrors the OpenAI-compatible shape so the model
|
||||
//! list can be expanded without code changes. Added 2026-04-24 after
|
||||
//! iter 5 hit repeated Ollama Cloud 502s on kimi-k2:1t — OpenRouter
|
||||
//! free-tier models give us a different provider backbone as fallback.
|
||||
//!
|
||||
//! Key sourcing priority:
|
||||
//! 1. Env var `OPENROUTER_API_KEY`
|
||||
//! 2. `/home/profit/.env` (LLM Team convention)
|
||||
//! 3. `/root/llm_team_config.json` → providers.openrouter.api_key
|
||||
//!
|
||||
//! First hit wins. Key is resolved once at gateway startup and stored
|
||||
//! on `V1State.openrouter_key`.
|
||||
|
||||
use std::time::Duration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
||||
|
||||
const OR_BASE_URL: &str = "https://openrouter.ai/api/v1";
|
||||
const OR_TIMEOUT_SECS: u64 = 180;
|
||||
|
||||
pub fn resolve_openrouter_key() -> Option<String> {
|
||||
if let Ok(k) = std::env::var("OPENROUTER_API_KEY") {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
// LLM Team UI writes its key to ~/.env on the host user — pick it up
|
||||
// from the same source so the free-tier rescue path works without
|
||||
// an explicit systemd Environment= line.
|
||||
for path in ["/home/profit/.env", "/root/.env"] {
|
||||
if let Ok(raw) = std::fs::read_to_string(path) {
|
||||
for line in raw.lines() {
|
||||
if let Some(rest) = line.strip_prefix("OPENROUTER_API_KEY=") {
|
||||
let k = rest.trim().trim_matches('"').trim_matches('\'');
|
||||
if !k.is_empty() { return Some(k.to_string()); }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(raw) = std::fs::read_to_string("/root/llm_team_config.json") {
|
||||
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) {
|
||||
if let Some(k) = v.pointer("/providers/openrouter/api_key").and_then(|x| x.as_str()) {
|
||||
if !k.trim().is_empty() { return Some(k.trim().to_string()); }
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn chat(
|
||||
key: &str,
|
||||
req: &ChatRequest,
|
||||
) -> Result<ChatResponse, String> {
|
||||
// Strip the "openrouter/" prefix if the caller used the namespaced
|
||||
// form so OpenRouter sees the raw model id (e.g. "openai/gpt-oss-120b:free").
|
||||
let model = req.model.strip_prefix("openrouter/").unwrap_or(&req.model).to_string();
|
||||
|
||||
let body = ORChatBody {
|
||||
model: model.clone(),
|
||||
messages: req.messages.iter().map(|m| ORMessage {
|
||||
role: m.role.clone(),
|
||||
content: m.content.clone(),
|
||||
}).collect(),
|
||||
max_tokens: req.max_tokens.unwrap_or(800),
|
||||
temperature: req.temperature.unwrap_or(0.3),
|
||||
stream: false,
|
||||
};
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(OR_TIMEOUT_SECS))
|
||||
.build()
|
||||
.map_err(|e| format!("build client: {e}"))?;
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let resp = client
|
||||
.post(format!("{}/chat/completions", OR_BASE_URL))
|
||||
.bearer_auth(key)
|
||||
// OpenRouter recommends Referer + Title for attribution; absent
|
||||
// headers do not fail the call but help us see our traffic in
|
||||
// their dashboard.
|
||||
.header("HTTP-Referer", "https://vcp.devop.live")
|
||||
.header("X-Title", "Lakehouse Scrum")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("openrouter.ai unreachable: {e}"))?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_else(|_| "?".into());
|
||||
return Err(format!("openrouter.ai {}: {}", status, body));
|
||||
}
|
||||
|
||||
let parsed: ORChatResponse = resp.json().await
|
||||
.map_err(|e| format!("invalid openrouter response: {e}"))?;
|
||||
|
||||
let latency_ms = t0.elapsed().as_millis();
|
||||
let choice = parsed.choices.into_iter().next()
|
||||
.ok_or_else(|| "openrouter returned no choices".to_string())?;
|
||||
let text = choice.message.content;
|
||||
|
||||
let prompt_tokens = parsed.usage.as_ref().map(|u| u.prompt_tokens).unwrap_or_else(|| {
|
||||
let chars: usize = req.messages.iter().map(|m| m.content.chars().count()).sum();
|
||||
((chars + 3) / 4) as u32
|
||||
});
|
||||
let completion_tokens = parsed.usage.as_ref().map(|u| u.completion_tokens).unwrap_or_else(|| {
|
||||
((text.chars().count() + 3) / 4) as u32
|
||||
});
|
||||
|
||||
tracing::info!(
|
||||
target: "v1.chat",
|
||||
provider = "openrouter",
|
||||
model = %model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
latency_ms = latency_ms as u64,
|
||||
"openrouter chat completed",
|
||||
);
|
||||
|
||||
Ok(ChatResponse {
|
||||
id: format!("chatcmpl-{}", chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
|
||||
object: "chat.completion",
|
||||
created: chrono::Utc::now().timestamp(),
|
||||
model,
|
||||
choices: vec![Choice {
|
||||
index: 0,
|
||||
message: Message { role: "assistant".into(), content: text },
|
||||
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
||||
}],
|
||||
usage: UsageBlock {
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// -- OpenRouter wire shapes (OpenAI-compatible) --
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ORChatBody {
|
||||
model: String,
|
||||
messages: Vec<ORMessage>,
|
||||
max_tokens: u32,
|
||||
temperature: f64,
|
||||
stream: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ORMessage { role: String, content: String }
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ORChatResponse {
|
||||
choices: Vec<ORChoice>,
|
||||
#[serde(default)]
|
||||
usage: Option<ORUsage>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ORChoice {
|
||||
message: ORMessageResp,
|
||||
#[serde(default)]
|
||||
finish_reason: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ORMessageResp { content: String }
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ORUsage { prompt_tokens: u32, completion_tokens: u32 }
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resolve_openrouter_key_does_not_panic() {
|
||||
// Smoke test — all three sources may or may not be set depending
|
||||
// on environment; just confirm the call returns cleanly.
|
||||
let _ = resolve_openrouter_key();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_body_serializes_to_openai_shape() {
|
||||
let body = ORChatBody {
|
||||
model: "openai/gpt-oss-120b:free".into(),
|
||||
messages: vec![
|
||||
ORMessage { role: "user".into(), content: "review this".into() },
|
||||
],
|
||||
max_tokens: 800,
|
||||
temperature: 0.3,
|
||||
stream: false,
|
||||
};
|
||||
let json = serde_json::to_string(&body).unwrap();
|
||||
assert!(json.contains("\"model\":\"openai/gpt-oss-120b:free\""));
|
||||
assert!(json.contains("\"messages\""));
|
||||
assert!(json.contains("\"max_tokens\":800"));
|
||||
assert!(json.contains("\"stream\":false"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_prefix_strip_preserves_unprefixed() {
|
||||
// If caller passes "openrouter/openai/gpt-oss-120b:free" we strip.
|
||||
// If caller passes "openai/gpt-oss-120b:free" unchanged, we keep.
|
||||
let cases = [
|
||||
("openrouter/openai/gpt-oss-120b:free", "openai/gpt-oss-120b:free"),
|
||||
("openai/gpt-oss-120b:free", "openai/gpt-oss-120b:free"),
|
||||
("google/gemma-3-27b-it:free", "google/gemma-3-27b-it:free"),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
let out = input.strip_prefix("openrouter/").unwrap_or(input);
|
||||
assert_eq!(out, expected, "{input} should become {expected}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//! `/v1/respond` — the **execution** API (distinct from `/v1/chat`, the
|
||||
//! completion API).
|
||||
//!
|
||||
//! This is the consolidation move called out in the 2026-04-23 session:
|
||||
//! lift the proven pipeline from `tests/multi-agent/orchestrator.ts`
|
||||
//! (executor → reviewer → escalate → validate → seal playbook →
|
||||
//! write-through to KB) into the gateway, so the production path has
|
||||
//! the intelligence the tests already proved.
|
||||
//!
|
||||
//! `/v1/chat` stays a naive completion proxy for callers that want one.
|
||||
//! `/v1/respond` is where the loop lives. Every orchestrator-style
|
||||
//! caller migrates here and the TS harnesses become thin clients.
|
||||
//!
|
||||
//! This file holds the HTTP surface + request/response shapes. The loop
|
||||
//! itself lives in `execution_loop::ExecutionLoop`.
|
||||
|
||||
use axum::{extract::State, http::StatusCode, Json};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::V1State;
|
||||
use crate::execution_loop::{ExecutionLoop, LogEntry, RespondOutcome};
|
||||
|
||||
/// A structured task — mirrors `TaskSpec` in `tests/multi-agent/agent.ts`.
|
||||
/// Kept deliberately open so non-staffing task classes (code-gen,
|
||||
/// DevOps-long-horizon) can land without a schema fight.
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct RespondRequest {
|
||||
/// Task class — routes to the right truth rules + validator. For the
|
||||
/// staffing substrate: `staffing.fill`, `staffing.rescue`,
|
||||
/// `staffing.sms_draft`. Truth-layer lookup is a no-op until a rule
|
||||
/// set is registered for the class.
|
||||
pub task_class: String,
|
||||
|
||||
/// Human-readable operation description — becomes the playbook
|
||||
/// `operation` field on seal, and the primary signal for
|
||||
/// playbook_memory embedding.
|
||||
pub operation: String,
|
||||
|
||||
/// Free-form structured context. Passed to the executor prompt and
|
||||
/// to the playbook seeder. Staffing tasks expect
|
||||
/// `{target_role, target_count, target_city, target_state, approach_hint}`
|
||||
/// but nothing here validates that — the validator crate will (Phase 43).
|
||||
#[serde(default)]
|
||||
pub spec: serde_json::Value,
|
||||
|
||||
/// Executor model. Defaults to the hot-path local model if omitted.
|
||||
/// See orchestrator.ts:28 (`EXECUTOR_MODEL = "qwen3.5:latest"`).
|
||||
#[serde(default)]
|
||||
pub executor_model: Option<String>,
|
||||
|
||||
/// Reviewer model. Defaults to the hot-path local reviewer.
|
||||
/// See orchestrator.ts:29 (`REVIEWER_MODEL = "qwen3:latest"`).
|
||||
#[serde(default)]
|
||||
pub reviewer_model: Option<String>,
|
||||
|
||||
/// Hard cap on executor turns. Default matches orchestrator.ts:30
|
||||
/// (`MAX_TURNS = 12`). Cloud escalation counts as a turn.
|
||||
#[serde(default)]
|
||||
pub max_turns: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct RespondResponse {
|
||||
/// `ok` = consensus reached, playbook sealed. `failed` = loop ran
|
||||
/// out of turns or hit the drift cap. `blocked` = truth-layer
|
||||
/// veto (Phase 42 rule citation in `error`).
|
||||
pub status: &'static str,
|
||||
|
||||
/// The final artifact — for staffing fills, `{fills: [{candidate_id, name}]}`.
|
||||
/// Empty on failure / block.
|
||||
pub artifact: serde_json::Value,
|
||||
|
||||
/// Structured cross-turn log. Same shape as orchestrator.ts LogEntry
|
||||
/// so existing tooling (kb extractors, fact_extractor.ts) reads it
|
||||
/// without change.
|
||||
pub log: Vec<LogEntry>,
|
||||
|
||||
/// Iteration count actually used. ≤ max_turns. Stamped on
|
||||
/// outcomes.jsonl per the indicator audit (2026-04-23).
|
||||
pub iterations: u32,
|
||||
|
||||
/// Error message on non-ok status. Truth-rule citations land here
|
||||
/// when `status == "blocked"`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn respond(
|
||||
State(state): State<V1State>,
|
||||
Json(req): Json<RespondRequest>,
|
||||
) -> Result<Json<RespondResponse>, (StatusCode, String)> {
|
||||
if req.operation.is_empty() {
|
||||
return Err((StatusCode::BAD_REQUEST, "operation must be non-empty".into()));
|
||||
}
|
||||
if req.task_class.is_empty() {
|
||||
return Err((StatusCode::BAD_REQUEST, "task_class must be non-empty".into()));
|
||||
}
|
||||
|
||||
let mut loop_runner = ExecutionLoop::new(state, req);
|
||||
let outcome = loop_runner.run().await.map_err(|e| {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, format!("execution loop: {e}"))
|
||||
})?;
|
||||
|
||||
let (status, error) = match &outcome {
|
||||
RespondOutcome::Ok { .. } => ("ok", None),
|
||||
RespondOutcome::Failed { reason, .. } => ("failed", Some(reason.clone())),
|
||||
RespondOutcome::Blocked { reason, .. } => ("blocked", Some(reason.clone())),
|
||||
};
|
||||
|
||||
Ok(Json(RespondResponse {
|
||||
status,
|
||||
artifact: outcome.artifact(),
|
||||
log: outcome.into_log(),
|
||||
iterations: loop_runner.turns_used(),
|
||||
error,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn respond_request_parses_minimal() {
|
||||
let raw = r#"{
|
||||
"task_class": "staffing.fill",
|
||||
"operation": "fill: Welder x2 in Toledo, OH"
|
||||
}"#;
|
||||
let r: RespondRequest = serde_json::from_str(raw).unwrap();
|
||||
assert_eq!(r.task_class, "staffing.fill");
|
||||
assert_eq!(r.executor_model, None);
|
||||
assert_eq!(r.max_turns, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn respond_request_parses_full() {
|
||||
let raw = r#"{
|
||||
"task_class": "staffing.fill",
|
||||
"operation": "fill: Welder x2 in Toledo, OH",
|
||||
"spec": {"target_role": "Welder", "target_count": 2, "target_city": "Toledo", "target_state": "OH"},
|
||||
"executor_model": "qwen3.5:latest",
|
||||
"reviewer_model": "qwen3:latest",
|
||||
"max_turns": 12
|
||||
}"#;
|
||||
let r: RespondRequest = serde_json::from_str(raw).unwrap();
|
||||
assert_eq!(r.executor_model.as_deref(), Some("qwen3.5:latest"));
|
||||
assert_eq!(r.max_turns, Some(12));
|
||||
assert_eq!(r.spec["target_count"], 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
use serde::Serialize;
|
||||
use truth::default_truth_store;
|
||||
|
||||
// Note: truth_router() was a stub wrapper around a single /context route
|
||||
// that nothing called — v1/mod.rs wires get(truth::context) directly
|
||||
// onto its own router. Removed 2026-04-24 along with its #[allow(dead_code)]
|
||||
// attribute; the handler below is the real surface.
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ContextResponse {
|
||||
pub task_classes: Vec<String>,
|
||||
pub rules: Vec<RuleInfo>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct RuleInfo {
|
||||
pub id: String,
|
||||
pub task_class: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
pub async fn context() -> axum::Json<ContextResponse> {
|
||||
let store = default_truth_store();
|
||||
|
||||
let task_classes: Vec<String> = vec![
|
||||
"staffing.fill".to_string(),
|
||||
"staffing.rescue".to_string(),
|
||||
"staffing.sms_draft".to_string(),
|
||||
"staffing.any".to_string(),
|
||||
];
|
||||
|
||||
let mut rules = Vec::new();
|
||||
for tc in &task_classes {
|
||||
for rule in store.get_rules(tc) {
|
||||
rules.push(RuleInfo {
|
||||
id: rule.id.clone(),
|
||||
task_class: rule.task_class.clone(),
|
||||
description: rule.description.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
axum::Json(ContextResponse {
|
||||
task_classes,
|
||||
rules,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "ingestd"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
storaged = { path = "../storaged" }
|
||||
catalogd = { path = "../catalogd" }
|
||||
vectord = { path = "../vectord" }
|
||||
journald = { path = "../journald" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true, features = ["multipart"] }
|
||||
lopdf = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
arrow = { workspace = true }
|
||||
parquet = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
csv = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
object_store = { workspace = true }
|
||||
tokio-postgres = { workspace = true }
|
||||
mysql_async = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
croner = "3"
|
||||
@@ -0,0 +1,203 @@
|
||||
use arrow::array::{ArrayRef, Float64Array, Int64Array, StringArray, BooleanArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Inferred column type from sampling data.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum InferredType {
|
||||
Integer,
|
||||
Float,
|
||||
Boolean,
|
||||
String,
|
||||
}
|
||||
|
||||
/// Parse CSV bytes into Arrow RecordBatches with automatic schema detection.
|
||||
/// Per ADR-010: ambiguous types default to String.
|
||||
pub fn parse_csv(content: &[u8]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
|
||||
let mut reader = csv::ReaderBuilder::new()
|
||||
.flexible(true) // allow varying column counts
|
||||
.trim(csv::Trim::All)
|
||||
.from_reader(content);
|
||||
|
||||
let headers: Vec<String> = reader.headers()
|
||||
.map_err(|e| format!("CSV header error: {e}"))?
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, h)| {
|
||||
let h = h.trim().to_string();
|
||||
if h.is_empty() { format!("column_{i}") } else { sanitize_column_name(&h) }
|
||||
})
|
||||
.collect();
|
||||
|
||||
let n_cols = headers.len();
|
||||
if n_cols == 0 {
|
||||
return Err("CSV has no columns".into());
|
||||
}
|
||||
|
||||
// Read all rows into string columns
|
||||
let mut columns: Vec<Vec<String>> = vec![vec![]; n_cols];
|
||||
let mut row_count = 0;
|
||||
|
||||
for result in reader.records() {
|
||||
let record = result.map_err(|e| format!("CSV row error: {e}"))?;
|
||||
for (i, field) in record.iter().enumerate() {
|
||||
if i < n_cols {
|
||||
columns[i].push(field.trim().to_string());
|
||||
}
|
||||
}
|
||||
// Pad short rows with empty strings
|
||||
for col in columns.iter_mut().skip(record.len().min(n_cols)) {
|
||||
col.push(String::new());
|
||||
}
|
||||
row_count += 1;
|
||||
}
|
||||
|
||||
if row_count == 0 {
|
||||
return Err("CSV has no data rows".into());
|
||||
}
|
||||
|
||||
tracing::info!("parsed CSV: {row_count} rows × {n_cols} columns");
|
||||
|
||||
// Infer types by sampling (look at all values)
|
||||
let types: Vec<InferredType> = columns.iter().map(|col| infer_column_type(col)).collect();
|
||||
|
||||
// Build Arrow schema
|
||||
let fields: Vec<Field> = headers.iter().zip(types.iter()).map(|(name, typ)| {
|
||||
let dt = match typ {
|
||||
InferredType::Integer => DataType::Int64,
|
||||
InferredType::Float => DataType::Float64,
|
||||
InferredType::Boolean => DataType::Boolean,
|
||||
InferredType::String => DataType::Utf8,
|
||||
};
|
||||
Field::new(name, dt, true) // all nullable
|
||||
}).collect();
|
||||
|
||||
let schema = Arc::new(Schema::new(fields));
|
||||
|
||||
// Build arrays
|
||||
let arrays: Vec<ArrayRef> = columns.iter().zip(types.iter()).map(|(col, typ)| {
|
||||
match typ {
|
||||
InferredType::Integer => {
|
||||
let vals: Vec<Option<i64>> = col.iter().map(|v| {
|
||||
if v.is_empty() { None } else { v.replace(',', "").parse().ok() }
|
||||
}).collect();
|
||||
Arc::new(Int64Array::from(vals)) as ArrayRef
|
||||
}
|
||||
InferredType::Float => {
|
||||
let vals: Vec<Option<f64>> = col.iter().map(|v| {
|
||||
if v.is_empty() { None }
|
||||
else { v.replace(',', "").replace('$', "").replace('%', "").parse().ok() }
|
||||
}).collect();
|
||||
Arc::new(Float64Array::from(vals)) as ArrayRef
|
||||
}
|
||||
InferredType::Boolean => {
|
||||
let vals: Vec<Option<bool>> = col.iter().map(|v| {
|
||||
match v.to_lowercase().as_str() {
|
||||
"true" | "yes" | "1" | "y" | "t" => Some(true),
|
||||
"false" | "no" | "0" | "n" | "f" => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}).collect();
|
||||
Arc::new(BooleanArray::from(vals)) as ArrayRef
|
||||
}
|
||||
InferredType::String => {
|
||||
Arc::new(StringArray::from(col.clone())) as ArrayRef
|
||||
}
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
Ok((schema, vec![batch]))
|
||||
}
|
||||
|
||||
/// Infer column type from values. Conservative: defaults to String on ambiguity.
|
||||
fn infer_column_type(values: &[String]) -> InferredType {
|
||||
let non_empty: Vec<&str> = values.iter()
|
||||
.map(|v| v.as_str())
|
||||
.filter(|v| !v.is_empty() && *v != "NULL" && *v != "null" && *v != "N/A" && *v != "n/a")
|
||||
.collect();
|
||||
|
||||
if non_empty.is_empty() {
|
||||
return InferredType::String;
|
||||
}
|
||||
|
||||
// Check boolean
|
||||
let all_bool = non_empty.iter().all(|v| {
|
||||
matches!(v.to_lowercase().as_str(), "true" | "false" | "yes" | "no" | "1" | "0" | "y" | "n" | "t" | "f")
|
||||
});
|
||||
if all_bool && non_empty.len() >= 2 {
|
||||
// Make sure it's not just all "1" and "0" which could be integers
|
||||
let has_text_bool = non_empty.iter().any(|v| {
|
||||
matches!(v.to_lowercase().as_str(), "true" | "false" | "yes" | "no" | "y" | "n" | "t" | "f")
|
||||
});
|
||||
if has_text_bool {
|
||||
return InferredType::Boolean;
|
||||
}
|
||||
}
|
||||
|
||||
// Check integer (allow commas as thousands separator)
|
||||
let int_rate = non_empty.iter()
|
||||
.filter(|v| v.replace(',', "").parse::<i64>().is_ok())
|
||||
.count() as f64 / non_empty.len() as f64;
|
||||
if int_rate > 0.95 {
|
||||
return InferredType::Integer;
|
||||
}
|
||||
|
||||
// Check float (allow $, %, commas)
|
||||
let float_rate = non_empty.iter()
|
||||
.filter(|v| v.replace(',', "").replace('$', "").replace('%', "").parse::<f64>().is_ok())
|
||||
.count() as f64 / non_empty.len() as f64;
|
||||
if float_rate > 0.95 {
|
||||
return InferredType::Float;
|
||||
}
|
||||
|
||||
InferredType::String
|
||||
}
|
||||
|
||||
/// Sanitize column name for SQL compatibility.
|
||||
fn sanitize_column_name(name: &str) -> String {
|
||||
name.chars()
|
||||
.map(|c| if c.is_alphanumeric() || c == '_' { c.to_ascii_lowercase() } else { '_' })
|
||||
.collect::<String>()
|
||||
.trim_matches('_')
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_simple_csv() {
|
||||
let csv = b"Name,Age,Salary\nAlice,30,50000\nBob,25,45000\n";
|
||||
let (schema, batches) = parse_csv(csv).unwrap();
|
||||
assert_eq!(schema.fields().len(), 3);
|
||||
assert_eq!(batches[0].num_rows(), 2);
|
||||
assert_eq!(schema.field(1).data_type(), &DataType::Int64);
|
||||
assert_eq!(schema.field(2).data_type(), &DataType::Int64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_csv_with_mixed_types() {
|
||||
let csv = b"id,value\n1,hello\n2,world\n3,N/A\n";
|
||||
let (schema, _) = parse_csv(csv).unwrap();
|
||||
assert_eq!(schema.field(0).data_type(), &DataType::Int64);
|
||||
assert_eq!(schema.field(1).data_type(), &DataType::Utf8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_csv_with_dollar_amounts() {
|
||||
let csv = b"item,price\nWidget,$29.99\nGadget,$149.50\n";
|
||||
let (schema, _) = parse_csv(csv).unwrap();
|
||||
assert_eq!(schema.field(1).data_type(), &DataType::Float64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_names() {
|
||||
assert_eq!(sanitize_column_name("First Name"), "first_name");
|
||||
assert_eq!(sanitize_column_name("Bill Rate ($)"), "bill_rate");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/// Database connector: pull tables from PostgreSQL directly into Parquet.
|
||||
/// Connects, reads schema, streams rows into Arrow RecordBatches, writes Parquet.
|
||||
/// No ORM, no schema definition — schema inferred from the database.
|
||||
|
||||
use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, StringArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use std::sync::Arc;
|
||||
use tokio_postgres::{Client, NoTls, types::Type as PgType};
|
||||
|
||||
/// Connection config for a database.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub struct DbConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub database: String,
|
||||
pub user: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
impl DbConfig {
|
||||
pub fn connection_string(&self) -> String {
|
||||
if self.password.is_empty() {
|
||||
format!(
|
||||
"host={} port={} dbname={} user={}",
|
||||
self.host, self.port, self.database, self.user
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"host={} port={} dbname={} user={} password={}",
|
||||
self.host, self.port, self.database, self.user, self.password
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a database table import.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct DbIngestResult {
|
||||
pub table_name: String,
|
||||
pub rows: usize,
|
||||
pub columns: usize,
|
||||
pub schema_detected: Vec<String>, // column name: type pairs
|
||||
}
|
||||
|
||||
/// Connect to PostgreSQL and import a table as Arrow RecordBatches.
|
||||
pub async fn import_postgres_table(
|
||||
config: &DbConfig,
|
||||
table_name: &str,
|
||||
limit: Option<usize>,
|
||||
) -> Result<(Arc<Schema>, Vec<RecordBatch>, DbIngestResult), String> {
|
||||
// Connect
|
||||
let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
|
||||
.await
|
||||
.map_err(|e| format!("postgres connect error: {e}"))?;
|
||||
|
||||
// Spawn connection handler
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
tracing::error!("postgres connection error: {e}");
|
||||
}
|
||||
});
|
||||
|
||||
tracing::info!("connected to postgres {}:{}/{}", config.host, config.port, config.database);
|
||||
|
||||
// Get column info
|
||||
let columns = get_table_columns(&client, table_name).await?;
|
||||
if columns.is_empty() {
|
||||
return Err(format!("table '{}' not found or has no columns", table_name));
|
||||
}
|
||||
|
||||
let schema_detected: Vec<String> = columns.iter()
|
||||
.map(|(name, pg_type)| format!("{}: {}", name, pg_type_to_string(pg_type)))
|
||||
.collect();
|
||||
|
||||
tracing::info!("table '{}': {} columns", table_name, columns.len());
|
||||
|
||||
// Build Arrow schema
|
||||
let arrow_fields: Vec<Field> = columns.iter()
|
||||
.map(|(name, pg_type)| Field::new(name, pg_type_to_arrow(pg_type), true))
|
||||
.collect();
|
||||
let schema = Arc::new(Schema::new(arrow_fields));
|
||||
|
||||
// Query rows
|
||||
let sql = match limit {
|
||||
Some(n) => format!("SELECT * FROM \"{}\" LIMIT {}", table_name, n),
|
||||
None => format!("SELECT * FROM \"{}\"", table_name),
|
||||
};
|
||||
|
||||
let rows = client.query(&sql, &[])
|
||||
.await
|
||||
.map_err(|e| format!("query error: {e}"))?;
|
||||
|
||||
let row_count = rows.len();
|
||||
tracing::info!("fetched {} rows from '{}'", row_count, table_name);
|
||||
|
||||
if row_count == 0 {
|
||||
return Ok((schema.clone(), vec![], DbIngestResult {
|
||||
table_name: table_name.to_string(),
|
||||
rows: 0,
|
||||
columns: columns.len(),
|
||||
schema_detected,
|
||||
}));
|
||||
}
|
||||
|
||||
// Convert to Arrow arrays
|
||||
let mut arrays: Vec<ArrayRef> = Vec::new();
|
||||
|
||||
for (col_idx, (_, pg_type)) in columns.iter().enumerate() {
|
||||
let array = rows_to_arrow_column(&rows, col_idx, pg_type)?;
|
||||
arrays.push(array);
|
||||
}
|
||||
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
let result = DbIngestResult {
|
||||
table_name: table_name.to_string(),
|
||||
rows: row_count,
|
||||
columns: columns.len(),
|
||||
schema_detected,
|
||||
};
|
||||
|
||||
Ok((schema, vec![batch], result))
|
||||
}
|
||||
|
||||
/// List all tables in the database.
|
||||
pub async fn list_postgres_tables(config: &DbConfig) -> Result<Vec<String>, String> {
|
||||
let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
|
||||
.await
|
||||
.map_err(|e| format!("connect error: {e}"))?;
|
||||
|
||||
tokio::spawn(async move { let _ = connection.await; });
|
||||
|
||||
let rows = client.query(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name",
|
||||
&[],
|
||||
).await.map_err(|e| format!("query error: {e}"))?;
|
||||
|
||||
Ok(rows.iter().map(|r| r.get::<_, String>(0)).collect())
|
||||
}
|
||||
|
||||
/// Get column names and types for a table.
|
||||
async fn get_table_columns(client: &Client, table_name: &str) -> Result<Vec<(String, PgType)>, String> {
|
||||
// Use a dummy query to get column types
|
||||
let stmt = client.prepare(&format!("SELECT * FROM \"{}\" LIMIT 0", table_name))
|
||||
.await
|
||||
.map_err(|e| format!("prepare error for '{}': {e}", table_name))?;
|
||||
|
||||
Ok(stmt.columns().iter()
|
||||
.map(|col| (col.name().to_string(), col.type_().clone()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Convert a Postgres type to Arrow DataType.
|
||||
fn pg_type_to_arrow(pg_type: &PgType) -> DataType {
|
||||
match *pg_type {
|
||||
PgType::BOOL => DataType::Boolean,
|
||||
PgType::INT2 | PgType::INT4 => DataType::Int32,
|
||||
PgType::INT8 | PgType::OID => DataType::Int64,
|
||||
PgType::FLOAT4 | PgType::FLOAT8 | PgType::NUMERIC => DataType::Float64,
|
||||
_ => DataType::Utf8, // everything else → string (safe default per ADR-010)
|
||||
}
|
||||
}
|
||||
|
||||
fn pg_type_to_string(pg_type: &PgType) -> String {
|
||||
format!("{}", pg_type)
|
||||
}
|
||||
|
||||
/// Convert a column of Postgres rows to an Arrow array.
|
||||
fn rows_to_arrow_column(
|
||||
rows: &[tokio_postgres::Row],
|
||||
col_idx: usize,
|
||||
pg_type: &PgType,
|
||||
) -> Result<ArrayRef, String> {
|
||||
match *pg_type {
|
||||
PgType::BOOL => {
|
||||
let vals: Vec<Option<bool>> = rows.iter()
|
||||
.map(|r| r.try_get(col_idx).ok())
|
||||
.collect();
|
||||
Ok(Arc::new(BooleanArray::from(vals)))
|
||||
}
|
||||
PgType::INT2 => {
|
||||
let vals: Vec<Option<i32>> = rows.iter()
|
||||
.map(|r| r.try_get::<_, i16>(col_idx).ok().map(|v| v as i32))
|
||||
.collect();
|
||||
Ok(Arc::new(Int32Array::from(vals)))
|
||||
}
|
||||
PgType::INT4 => {
|
||||
let vals: Vec<Option<i32>> = rows.iter()
|
||||
.map(|r| r.try_get(col_idx).ok())
|
||||
.collect();
|
||||
Ok(Arc::new(Int32Array::from(vals)))
|
||||
}
|
||||
PgType::INT8 | PgType::OID => {
|
||||
let vals: Vec<Option<i64>> = rows.iter()
|
||||
.map(|r| r.try_get(col_idx).ok())
|
||||
.collect();
|
||||
Ok(Arc::new(Int64Array::from(vals)))
|
||||
}
|
||||
PgType::FLOAT4 => {
|
||||
let vals: Vec<Option<f64>> = rows.iter()
|
||||
.map(|r| r.try_get::<_, f32>(col_idx).ok().map(|v| v as f64))
|
||||
.collect();
|
||||
Ok(Arc::new(Float64Array::from(vals)))
|
||||
}
|
||||
PgType::FLOAT8 => {
|
||||
let vals: Vec<Option<f64>> = rows.iter()
|
||||
.map(|r| r.try_get(col_idx).ok())
|
||||
.collect();
|
||||
Ok(Arc::new(Float64Array::from(vals)))
|
||||
}
|
||||
_ => {
|
||||
// Default: try to get as string
|
||||
let vals: Vec<Option<String>> = rows.iter()
|
||||
.map(|r| {
|
||||
r.try_get::<_, String>(col_idx).ok()
|
||||
.or_else(|| r.try_get::<_, serde_json::Value>(col_idx).ok().map(|v| v.to_string()))
|
||||
.or_else(|| Some("".to_string()))
|
||||
})
|
||||
.collect();
|
||||
Ok(Arc::new(StringArray::from(vals)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Detected file type from content inspection.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum FileType {
|
||||
Csv,
|
||||
Json,
|
||||
NdJson, // newline-delimited JSON
|
||||
Pdf,
|
||||
Text, // plain text, SMS logs, etc.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Detect file type from filename extension and content sniffing.
|
||||
pub fn detect_file_type(filename: &str, content: &[u8]) -> FileType {
|
||||
// Extension-based first
|
||||
let lower = filename.to_lowercase();
|
||||
if lower.ends_with(".csv") || lower.ends_with(".tsv") {
|
||||
return FileType::Csv;
|
||||
}
|
||||
if lower.ends_with(".json") {
|
||||
// Check if it's newline-delimited JSON
|
||||
if content.iter().take(4096).filter(|&&b| b == b'\n').count() > 2 {
|
||||
let first_line = content.split(|&b| b == b'\n').next().unwrap_or(b"");
|
||||
if first_line.starts_with(b"{") {
|
||||
return FileType::NdJson;
|
||||
}
|
||||
}
|
||||
return FileType::Json;
|
||||
}
|
||||
if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") {
|
||||
return FileType::NdJson;
|
||||
}
|
||||
if lower.ends_with(".pdf") {
|
||||
return FileType::Pdf;
|
||||
}
|
||||
if lower.ends_with(".txt") || lower.ends_with(".log") || lower.ends_with(".sms") {
|
||||
return FileType::Text;
|
||||
}
|
||||
|
||||
// Content sniffing fallback
|
||||
if content.starts_with(b"%PDF") {
|
||||
return FileType::Pdf;
|
||||
}
|
||||
if content.starts_with(b"[") || content.starts_with(b"{") {
|
||||
return FileType::Json;
|
||||
}
|
||||
|
||||
// Check if it looks like CSV (has commas and newlines in first chunk)
|
||||
let sample = &content[..content.len().min(4096)];
|
||||
let comma_count = sample.iter().filter(|&&b| b == b',').count();
|
||||
let newline_count = sample.iter().filter(|&&b| b == b'\n').count();
|
||||
if comma_count > 3 && newline_count > 1 {
|
||||
return FileType::Csv;
|
||||
}
|
||||
|
||||
// If it's valid UTF-8, treat as text
|
||||
if std::str::from_utf8(sample).is_ok() {
|
||||
return FileType::Text;
|
||||
}
|
||||
|
||||
FileType::Unknown
|
||||
}
|
||||
|
||||
/// Compute SHA-256 hash of content for deduplication.
|
||||
pub fn content_hash(content: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(content);
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn detect_csv_by_extension() {
|
||||
assert_eq!(detect_file_type("data.csv", b"a,b,c\n1,2,3"), FileType::Csv);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_json_by_extension() {
|
||||
assert_eq!(detect_file_type("data.json", b"[{\"a\":1}]"), FileType::Json);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_pdf_by_magic() {
|
||||
assert_eq!(detect_file_type("unknown", b"%PDF-1.4 blah"), FileType::Pdf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_csv_by_content() {
|
||||
let csv = b"name,age,city\nAlice,30,NYC\nBob,25,LA\n";
|
||||
assert_eq!(detect_file_type("unknown.dat", csv), FileType::Csv);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_hash_deterministic() {
|
||||
let h1 = content_hash(b"hello world");
|
||||
let h2 = content_hash(b"hello world");
|
||||
assert_eq!(h1, h2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
use arrow::array::{ArrayRef, Float64Array, Int64Array, StringArray, BooleanArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Parse JSON (array of objects or newline-delimited) into Arrow RecordBatches.
|
||||
/// Nested objects are flattened: {"a": {"b": 1}} → column "a_b".
|
||||
pub fn parse_json(content: &[u8]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
|
||||
let text = std::str::from_utf8(content).map_err(|e| format!("invalid UTF-8: {e}"))?;
|
||||
let text = text.trim();
|
||||
|
||||
// Parse into Vec<Map>
|
||||
let rows: Vec<serde_json::Map<String, serde_json::Value>> = if text.starts_with('[') {
|
||||
// JSON array
|
||||
let arr: Vec<serde_json::Value> = serde_json::from_str(text)
|
||||
.map_err(|e| format!("JSON parse error: {e}"))?;
|
||||
arr.into_iter()
|
||||
.filter_map(|v| v.as_object().cloned())
|
||||
.collect()
|
||||
} else {
|
||||
// Newline-delimited JSON
|
||||
text.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.map(|l| {
|
||||
let v: serde_json::Value = serde_json::from_str(l)
|
||||
.map_err(|e| format!("NDJSON line parse error: {e}"))?;
|
||||
v.as_object().cloned().ok_or_else(|| "NDJSON line is not an object".into())
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?
|
||||
};
|
||||
|
||||
if rows.is_empty() {
|
||||
return Err("JSON has no records".into());
|
||||
}
|
||||
|
||||
// Flatten and collect all columns
|
||||
let mut columns: BTreeMap<String, Vec<serde_json::Value>> = BTreeMap::new();
|
||||
let n_rows = rows.len();
|
||||
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
let flat = flatten_object(row, "");
|
||||
// Pad existing columns that aren't in this row
|
||||
for key in columns.keys().cloned().collect::<Vec<_>>() {
|
||||
if !flat.contains_key(&key) {
|
||||
columns.get_mut(&key).unwrap().push(serde_json::Value::Null);
|
||||
}
|
||||
}
|
||||
// Add new columns or append values
|
||||
for (key, val) in flat {
|
||||
let col = columns.entry(key).or_insert_with(Vec::new);
|
||||
// Pad with nulls if this column is new and we've seen prior rows
|
||||
while col.len() < row_idx {
|
||||
col.push(serde_json::Value::Null);
|
||||
}
|
||||
col.push(val);
|
||||
}
|
||||
}
|
||||
|
||||
// Pad short columns
|
||||
for col in columns.values_mut() {
|
||||
while col.len() < n_rows {
|
||||
col.push(serde_json::Value::Null);
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("parsed JSON: {n_rows} rows × {} columns", columns.len());
|
||||
|
||||
// Infer types and build schema
|
||||
let mut fields = Vec::new();
|
||||
let mut arrays: Vec<ArrayRef> = Vec::new();
|
||||
|
||||
for (name, values) in &columns {
|
||||
let (dt, array) = json_column_to_arrow(values);
|
||||
fields.push(Field::new(name, dt, true));
|
||||
arrays.push(array);
|
||||
}
|
||||
|
||||
let schema = Arc::new(Schema::new(fields));
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
Ok((schema, vec![batch]))
|
||||
}
|
||||
|
||||
/// Flatten nested JSON object: {"a": {"b": 1}} → {"a_b": 1}
|
||||
fn flatten_object(obj: &serde_json::Map<String, serde_json::Value>, prefix: &str) -> BTreeMap<String, serde_json::Value> {
|
||||
let mut result = BTreeMap::new();
|
||||
for (key, val) in obj {
|
||||
let full_key = if prefix.is_empty() { key.clone() } else { format!("{prefix}_{key}") };
|
||||
match val {
|
||||
serde_json::Value::Object(inner) => {
|
||||
result.extend(flatten_object(inner, &full_key));
|
||||
}
|
||||
other => {
|
||||
result.insert(full_key, other.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Convert a column of JSON values to an Arrow array.
|
||||
fn json_column_to_arrow(values: &[serde_json::Value]) -> (DataType, ArrayRef) {
|
||||
// Check if all non-null values are the same type
|
||||
let non_null: Vec<&serde_json::Value> = values.iter()
|
||||
.filter(|v| !v.is_null())
|
||||
.collect();
|
||||
|
||||
if non_null.is_empty() {
|
||||
return (DataType::Utf8, Arc::new(StringArray::from(vec![None::<&str>; values.len()])));
|
||||
}
|
||||
|
||||
let all_bool = non_null.iter().all(|v| v.is_boolean());
|
||||
let all_i64 = non_null.iter().all(|v| v.is_i64() || v.is_u64());
|
||||
let all_f64 = non_null.iter().all(|v| v.is_number());
|
||||
|
||||
if all_bool {
|
||||
let arr: Vec<Option<bool>> = values.iter().map(|v| v.as_bool()).collect();
|
||||
(DataType::Boolean, Arc::new(BooleanArray::from(arr)))
|
||||
} else if all_i64 {
|
||||
let arr: Vec<Option<i64>> = values.iter().map(|v| v.as_i64()).collect();
|
||||
(DataType::Int64, Arc::new(Int64Array::from(arr)))
|
||||
} else if all_f64 {
|
||||
let arr: Vec<Option<f64>> = values.iter().map(|v| v.as_f64()).collect();
|
||||
(DataType::Float64, Arc::new(Float64Array::from(arr)))
|
||||
} else {
|
||||
// Default to string
|
||||
let arr: Vec<Option<String>> = values.iter().map(|v| {
|
||||
match v {
|
||||
serde_json::Value::Null => None,
|
||||
serde_json::Value::String(s) => Some(s.clone()),
|
||||
other => Some(other.to_string()),
|
||||
}
|
||||
}).collect();
|
||||
(DataType::Utf8, Arc::new(StringArray::from(arr)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_json_array() {
|
||||
let json = br#"[{"name":"Alice","age":30},{"name":"Bob","age":25}]"#;
|
||||
let (schema, batches) = parse_json(json).unwrap();
|
||||
assert_eq!(batches[0].num_rows(), 2);
|
||||
assert!(schema.field_with_name("age").unwrap().data_type() == &DataType::Int64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_ndjson() {
|
||||
let json = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
|
||||
let (_, batches) = parse_json(json).unwrap();
|
||||
assert_eq!(batches[0].num_rows(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flatten_nested() {
|
||||
let json = br#"[{"user":{"name":"Alice","address":{"city":"NYC"}},"score":9.5}]"#;
|
||||
let (schema, _) = parse_json(json).unwrap();
|
||||
let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
|
||||
assert!(names.contains(&"user_name"));
|
||||
assert!(names.contains(&"user_address_city"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
pub mod db_ingest;
|
||||
pub mod my_stream;
|
||||
pub mod pg_stream;
|
||||
pub mod schedule;
|
||||
pub mod detect;
|
||||
pub mod csv_ingest;
|
||||
pub mod json_ingest;
|
||||
pub mod pdf_ingest;
|
||||
pub mod pipeline;
|
||||
pub mod schema_evolution;
|
||||
pub mod service;
|
||||
pub mod watcher;
|
||||
@@ -0,0 +1,410 @@
|
||||
/// Streaming MySQL ingest.
|
||||
///
|
||||
/// Mirrors `pg_stream` for MySQL sources. Same OFFSET-paginated strategy,
|
||||
/// same Parquet-streaming shape. Uses `mysql_async` (pure-rust) so we
|
||||
/// don't need a C client library at build time.
|
||||
///
|
||||
/// Type mapping follows ADR-010 (default to string on ambiguity):
|
||||
/// - Booleans (TINYINT(1)) and integer types map to Arrow Int32/Int64
|
||||
/// - Floating point and decimals → Float64
|
||||
/// - Everything else (text, varchar, json, date, timestamp) → Utf8
|
||||
///
|
||||
/// What's deliberately not supported (yet):
|
||||
/// - TLS connections — `minimal` feature is plain TCP only. Upgrade
|
||||
/// when a tenant actually needs it.
|
||||
/// - Keyset pagination — OFFSET scans are O(N²) at multi-million-row
|
||||
/// scale; same caveat as `pg_stream`.
|
||||
/// - BINARY/BLOB columns — currently rendered as base64 or empty string
|
||||
/// via Display-fallback.
|
||||
|
||||
use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use mysql_async::prelude::*;
|
||||
use mysql_async::{Conn, Opts, Row, Value};
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Request shape for MySQL streaming ingest.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub struct MyStreamRequest {
|
||||
/// mysql://user:pass@host:port/db
|
||||
pub dsn: String,
|
||||
pub table: String,
|
||||
#[serde(default)]
|
||||
pub dataset_name: Option<String>,
|
||||
/// Rows per fetch. Default 10_000.
|
||||
#[serde(default)]
|
||||
pub batch_size: Option<usize>,
|
||||
/// Column to ORDER BY for stable pagination. If omitted, the first
|
||||
/// column returned by the schema probe is used.
|
||||
#[serde(default)]
|
||||
pub order_by: Option<String>,
|
||||
/// Hard cap on total rows (for sampling / previews).
|
||||
#[serde(default)]
|
||||
pub limit: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct MyStreamResult {
|
||||
pub table: String,
|
||||
pub rows: usize,
|
||||
pub batches: usize,
|
||||
pub columns: usize,
|
||||
pub schema: Vec<String>,
|
||||
pub parquet_bytes: u64,
|
||||
pub duration_secs: f32,
|
||||
}
|
||||
|
||||
/// Parsed DSN pieces. Kept local (rather than reusing pg's DbConfig) so
|
||||
/// the MySQL connector doesn't depend on the PG path.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MyConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub user: String,
|
||||
pub password: String,
|
||||
pub database: String,
|
||||
}
|
||||
|
||||
impl MyConfig {
|
||||
/// Build a mysql_async `Opts` from the parsed config.
|
||||
pub fn to_opts(&self) -> Opts {
|
||||
let url = if self.password.is_empty() {
|
||||
format!(
|
||||
"mysql://{}@{}:{}/{}",
|
||||
percent(&self.user), percent(&self.host), self.port, percent(&self.database),
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"mysql://{}:{}@{}:{}/{}",
|
||||
percent(&self.user), percent(&self.password),
|
||||
percent(&self.host), self.port, percent(&self.database),
|
||||
)
|
||||
};
|
||||
Opts::from_url(&url).expect("MyConfig.to_opts: rebuilt URL should parse")
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal URL-encoder for the few characters that commonly appear in
|
||||
/// MySQL passwords. mysql_async's URL parser expects valid URL-encoded
|
||||
/// components.
|
||||
fn percent(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'@' | ':' | '/' | '?' | '#' | '[' | ']' | ' ' | '%' => {
|
||||
out.push_str(&format!("%{:02X}", c as u32));
|
||||
}
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse a mysql:// DSN.
|
||||
/// Supports: mysql://[user[:password]@]host[:port][/db]
|
||||
pub fn parse_dsn(dsn: &str) -> Result<MyConfig, String> {
|
||||
let rest = dsn
|
||||
.strip_prefix("mysql://")
|
||||
.ok_or_else(|| "DSN must start with mysql://".to_string())?;
|
||||
|
||||
let (auth_host, database) = match rest.split_once('/') {
|
||||
Some((ah, db)) => (ah, db.split('?').next().unwrap_or(db).to_string()),
|
||||
None => (rest, String::new()),
|
||||
};
|
||||
|
||||
let (userpass, hostport) = match auth_host.rsplit_once('@') {
|
||||
Some((up, hp)) => (Some(up), hp),
|
||||
None => (None, auth_host),
|
||||
};
|
||||
|
||||
let (user, password) = match userpass {
|
||||
Some(up) => match up.split_once(':') {
|
||||
Some((u, p)) => (u.to_string(), p.to_string()),
|
||||
None => (up.to_string(), String::new()),
|
||||
},
|
||||
None => ("root".to_string(), String::new()),
|
||||
};
|
||||
|
||||
let (host, port) = match hostport.rsplit_once(':') {
|
||||
Some((h, p)) => (
|
||||
h.to_string(),
|
||||
p.parse::<u16>().map_err(|_| format!("invalid port in DSN: {p}"))?,
|
||||
),
|
||||
None => (hostport.to_string(), 3306),
|
||||
};
|
||||
|
||||
if host.is_empty() {
|
||||
return Err("DSN has no host".into());
|
||||
}
|
||||
if database.is_empty() {
|
||||
return Err("DSN has no database path (mysql://... /db)".into());
|
||||
}
|
||||
|
||||
Ok(MyConfig { host, port, user, password, database })
|
||||
}
|
||||
|
||||
/// Stream a MySQL table as Parquet bytes.
|
||||
pub async fn stream_table_to_parquet(
|
||||
req: &MyStreamRequest,
|
||||
) -> Result<(bytes::Bytes, MyStreamResult), String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
let config = parse_dsn(&req.dsn)?;
|
||||
let batch_size = req.batch_size.unwrap_or(10_000).max(1);
|
||||
|
||||
let mut conn = Conn::new(config.to_opts()).await
|
||||
.map_err(|e| format!("mysql connect: {e}"))?;
|
||||
|
||||
// Probe columns via information_schema — gives real type names that
|
||||
// we can map to Arrow dtypes without needing to fetch a row first.
|
||||
let columns = probe_columns(&mut conn, &config.database, &req.table).await?;
|
||||
if columns.is_empty() {
|
||||
return Err(format!("table '{}' not found or has no columns", req.table));
|
||||
}
|
||||
|
||||
let arrow_fields: Vec<Field> = columns
|
||||
.iter()
|
||||
.map(|(name, ty)| Field::new(name, mysql_type_to_arrow(ty), true))
|
||||
.collect();
|
||||
let schema = Arc::new(Schema::new(arrow_fields));
|
||||
let schema_report: Vec<String> = columns
|
||||
.iter()
|
||||
.map(|(n, t)| format!("{}:{}", n, t))
|
||||
.collect();
|
||||
|
||||
let order_col = req.order_by.clone().unwrap_or_else(|| columns[0].0.clone());
|
||||
|
||||
let mut buf: Vec<u8> = Vec::with_capacity(1024 * 1024);
|
||||
let mut writer = ArrowWriter::try_new(&mut buf, schema.clone(), None)
|
||||
.map_err(|e| format!("ArrowWriter init: {e}"))?;
|
||||
|
||||
let mut total_rows: usize = 0;
|
||||
let mut batch_count: usize = 0;
|
||||
let row_cap = req.limit.unwrap_or(usize::MAX);
|
||||
|
||||
loop {
|
||||
let remaining = row_cap.saturating_sub(total_rows);
|
||||
if remaining == 0 { break; }
|
||||
let fetch = remaining.min(batch_size);
|
||||
|
||||
// Backticks are MySQL's identifier quote. Forbid backticks in
|
||||
// table/column names to prevent injection — neither pg nor us
|
||||
// should be accepting those anyway.
|
||||
if req.table.contains('`') || order_col.contains('`') {
|
||||
return Err("table or order_by column contains backticks — refused".into());
|
||||
}
|
||||
let sql = format!(
|
||||
"SELECT * FROM `{}` ORDER BY `{}` LIMIT {} OFFSET {}",
|
||||
req.table, order_col, fetch, total_rows,
|
||||
);
|
||||
|
||||
let rows: Vec<Row> = conn.query(&sql).await
|
||||
.map_err(|e| format!("fetch batch at offset {total_rows}: {e}"))?;
|
||||
|
||||
if rows.is_empty() { break; }
|
||||
let n = rows.len();
|
||||
|
||||
let arrays: Vec<ArrayRef> = columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, (_, ty))| rows_to_column(&rows, idx, ty))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch: {e}"))?;
|
||||
writer.write(&batch).map_err(|e| format!("ArrowWriter::write: {e}"))?;
|
||||
|
||||
total_rows += n;
|
||||
batch_count += 1;
|
||||
tracing::info!(
|
||||
"mysql stream '{}': fetched batch {} ({} rows, total {})",
|
||||
req.table, batch_count, n, total_rows,
|
||||
);
|
||||
|
||||
if n < fetch { break; }
|
||||
}
|
||||
|
||||
writer.close().map_err(|e| format!("ArrowWriter::close: {e}"))?;
|
||||
conn.disconnect().await.ok();
|
||||
|
||||
let result = MyStreamResult {
|
||||
table: req.table.clone(),
|
||||
rows: total_rows,
|
||||
batches: batch_count,
|
||||
columns: columns.len(),
|
||||
schema: schema_report,
|
||||
parquet_bytes: buf.len() as u64,
|
||||
duration_secs: t0.elapsed().as_secs_f32(),
|
||||
};
|
||||
|
||||
Ok((bytes::Bytes::from(buf), result))
|
||||
}
|
||||
|
||||
async fn probe_columns(
|
||||
conn: &mut Conn,
|
||||
schema: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<(String, String)>, String> {
|
||||
let sql = format!(
|
||||
"SELECT COLUMN_NAME, DATA_TYPE FROM information_schema.columns \
|
||||
WHERE table_schema = '{}' AND table_name = '{}' \
|
||||
ORDER BY ORDINAL_POSITION",
|
||||
schema.replace('\'', "''"),
|
||||
table.replace('\'', "''"),
|
||||
);
|
||||
let rows: Vec<(String, String)> = conn.query(&sql).await
|
||||
.map_err(|e| format!("probe columns: {e}"))?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// MySQL data type string → Arrow DataType. Conservative: anything we
|
||||
/// don't recognize becomes Utf8 (ADR-010).
|
||||
fn mysql_type_to_arrow(ty: &str) -> DataType {
|
||||
match ty.to_lowercase().as_str() {
|
||||
"tinyint" | "smallint" | "mediumint" | "int" | "integer" => DataType::Int32,
|
||||
"bigint" => DataType::Int64,
|
||||
"float" | "double" | "decimal" | "numeric" | "real" => DataType::Float64,
|
||||
"bit" | "bool" | "boolean" => DataType::Boolean,
|
||||
_ => DataType::Utf8,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a single column slice of MySQL rows into an Arrow array.
|
||||
fn rows_to_column(
|
||||
rows: &[Row],
|
||||
idx: usize,
|
||||
ty: &str,
|
||||
) -> Result<ArrayRef, String> {
|
||||
let arrow_ty = mysql_type_to_arrow(ty);
|
||||
match arrow_ty {
|
||||
DataType::Boolean => {
|
||||
let v: Vec<Option<bool>> = rows.iter().map(|r| cell_as_bool(r, idx)).collect();
|
||||
Ok(Arc::new(BooleanArray::from(v)))
|
||||
}
|
||||
DataType::Int32 => {
|
||||
let v: Vec<Option<i32>> = rows.iter().map(|r| cell_as_i64(r, idx).map(|n| n as i32)).collect();
|
||||
Ok(Arc::new(Int32Array::from(v)))
|
||||
}
|
||||
DataType::Int64 => {
|
||||
let v: Vec<Option<i64>> = rows.iter().map(|r| cell_as_i64(r, idx)).collect();
|
||||
Ok(Arc::new(Int64Array::from(v)))
|
||||
}
|
||||
DataType::Float64 => {
|
||||
let v: Vec<Option<f64>> = rows.iter().map(|r| cell_as_f64(r, idx)).collect();
|
||||
Ok(Arc::new(Float64Array::from(v)))
|
||||
}
|
||||
_ => {
|
||||
let v: Vec<Option<String>> = rows.iter().map(|r| cell_as_string(r, idx)).collect();
|
||||
Ok(Arc::new(StringArray::from(v)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cell(r: &Row, idx: usize) -> &Value { r.as_ref(idx).unwrap_or(&Value::NULL) }
|
||||
|
||||
fn cell_as_bool(r: &Row, idx: usize) -> Option<bool> {
|
||||
match cell(r, idx) {
|
||||
Value::NULL => None,
|
||||
Value::Int(n) => Some(*n != 0),
|
||||
Value::UInt(n) => Some(*n != 0),
|
||||
Value::Bytes(b) => std::str::from_utf8(b).ok().and_then(|s| {
|
||||
match s.to_ascii_lowercase().as_str() {
|
||||
"true" | "1" | "y" | "yes" => Some(true),
|
||||
"false" | "0" | "n" | "no" => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn cell_as_i64(r: &Row, idx: usize) -> Option<i64> {
|
||||
match cell(r, idx) {
|
||||
Value::NULL => None,
|
||||
Value::Int(n) => Some(*n),
|
||||
Value::UInt(n) => i64::try_from(*n).ok(),
|
||||
Value::Float(f) => Some(*f as i64),
|
||||
Value::Double(f) => Some(*f as i64),
|
||||
Value::Bytes(b) => std::str::from_utf8(b).ok().and_then(|s| s.parse().ok()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn cell_as_f64(r: &Row, idx: usize) -> Option<f64> {
|
||||
match cell(r, idx) {
|
||||
Value::NULL => None,
|
||||
Value::Int(n) => Some(*n as f64),
|
||||
Value::UInt(n) => Some(*n as f64),
|
||||
Value::Float(f) => Some(*f as f64),
|
||||
Value::Double(f) => Some(*f),
|
||||
Value::Bytes(b) => std::str::from_utf8(b).ok().and_then(|s| s.parse().ok()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn cell_as_string(r: &Row, idx: usize) -> Option<String> {
|
||||
match cell(r, idx) {
|
||||
Value::NULL => None,
|
||||
Value::Bytes(b) => Some(String::from_utf8_lossy(b).into_owned()),
|
||||
Value::Int(n) => Some(n.to_string()),
|
||||
Value::UInt(n) => Some(n.to_string()),
|
||||
Value::Float(f) => Some(f.to_string()),
|
||||
Value::Double(f) => Some(f.to_string()),
|
||||
Value::Date(y, mo, d, h, mi, s, _us) => {
|
||||
Some(format!("{:04}-{:02}-{:02} {:02}:{:02}:{:02}", y, mo, d, h, mi, s))
|
||||
}
|
||||
Value::Time(neg, days, h, mi, s, _us) => {
|
||||
let sign = if *neg { "-" } else { "" };
|
||||
Some(format!("{}{}d {:02}:{:02}:{:02}", sign, days, h, mi, s))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_full() {
|
||||
let c = parse_dsn("mysql://daisy:secret@db.example.com:3307/my_db").unwrap();
|
||||
assert_eq!(c.host, "db.example.com");
|
||||
assert_eq!(c.port, 3307);
|
||||
assert_eq!(c.user, "daisy");
|
||||
assert_eq!(c.password, "secret");
|
||||
assert_eq!(c.database, "my_db");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_default_port() {
|
||||
let c = parse_dsn("mysql://root@localhost/shop").unwrap();
|
||||
assert_eq!(c.port, 3306);
|
||||
assert_eq!(c.user, "root");
|
||||
assert_eq!(c.password, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_no_auth() {
|
||||
let c = parse_dsn("mysql://127.0.0.1:3306/analytics").unwrap();
|
||||
assert_eq!(c.user, "root");
|
||||
assert_eq!(c.host, "127.0.0.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_rejects_non_mysql() {
|
||||
assert!(parse_dsn("postgresql://host/db").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_requires_db() {
|
||||
assert!(parse_dsn("mysql://localhost:3306").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_map_basics() {
|
||||
assert_eq!(mysql_type_to_arrow("int"), DataType::Int32);
|
||||
assert_eq!(mysql_type_to_arrow("BIGINT"), DataType::Int64);
|
||||
assert_eq!(mysql_type_to_arrow("decimal"), DataType::Float64);
|
||||
assert_eq!(mysql_type_to_arrow("varchar"), DataType::Utf8);
|
||||
assert_eq!(mysql_type_to_arrow("json"), DataType::Utf8);
|
||||
assert_eq!(mysql_type_to_arrow("bool"), DataType::Boolean);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
use arrow::array::{ArrayRef, Int32Array, StringArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Extract text from a PDF file.
|
||||
///
|
||||
/// Two-tier approach:
|
||||
/// 1. **Text extraction (lopdf)** — fast, works on digital/native PDFs.
|
||||
/// Returns one row per page: (source_file, page_number, text_content).
|
||||
/// 2. **OCR fallback (Tesseract)** — if text extraction yields zero
|
||||
/// pages (common with scanned documents, image-only PDFs, or PDFs
|
||||
/// exported from scanners without an OCR layer), we extract embedded
|
||||
/// images from the PDF and run Tesseract on each. Falls back
|
||||
/// gracefully if Tesseract isn't installed (returns an error
|
||||
/// directing the operator to install it).
|
||||
///
|
||||
/// Schema is identical for both paths so downstream consumers (chunker,
|
||||
/// vectord, queryd) don't need to know how the text was produced.
|
||||
pub fn parse_pdf(content: &[u8], source_filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
|
||||
let doc = lopdf::Document::load_mem(content)
|
||||
.map_err(|e| format!("PDF load error: {e}"))?;
|
||||
|
||||
let pages = doc.get_pages();
|
||||
let mut page_numbers: Vec<i32> = Vec::new();
|
||||
let mut page_texts: Vec<String> = Vec::new();
|
||||
let mut sources: Vec<String> = Vec::new();
|
||||
|
||||
for (&page_num, _) in pages.iter() {
|
||||
let text = doc.extract_text(&[page_num]).unwrap_or_default();
|
||||
let text = text.trim().to_string();
|
||||
|
||||
if !text.is_empty() {
|
||||
page_numbers.push(page_num as i32);
|
||||
page_texts.push(text);
|
||||
sources.push(source_filename.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 2: OCR fallback for scanned pages.
|
||||
if page_numbers.is_empty() {
|
||||
tracing::info!(
|
||||
"no extractable text in '{}' — attempting OCR on {} pages",
|
||||
source_filename, pages.len(),
|
||||
);
|
||||
let ocr_results = ocr_pdf_images(&doc, source_filename)?;
|
||||
if ocr_results.is_empty() {
|
||||
return Err(
|
||||
"PDF contains no extractable text and OCR produced no output. \
|
||||
Ensure tesseract is installed (`apt install tesseract-ocr tesseract-ocr-eng`)."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
for (pg, text) in ocr_results {
|
||||
page_numbers.push(pg);
|
||||
page_texts.push(text);
|
||||
sources.push(source_filename.to_string());
|
||||
}
|
||||
tracing::info!(
|
||||
"OCR recovered {} pages from '{}'", page_numbers.len(), source_filename,
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
"extracted {} pages with text from PDF '{}'",
|
||||
page_numbers.len(), source_filename,
|
||||
);
|
||||
}
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("source_file", DataType::Utf8, false),
|
||||
Field::new("page_number", DataType::Int32, false),
|
||||
Field::new("text_content", DataType::Utf8, false),
|
||||
]));
|
||||
|
||||
let arrays: Vec<ArrayRef> = vec![
|
||||
Arc::new(StringArray::from(sources)),
|
||||
Arc::new(Int32Array::from(page_numbers)),
|
||||
Arc::new(StringArray::from(page_texts)),
|
||||
];
|
||||
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
Ok((schema, vec![batch]))
|
||||
}
|
||||
|
||||
/// Extract embedded images from the PDF, write each to a temp file,
|
||||
/// run Tesseract, collect the OCR'd text. Returns (page_number, text)
|
||||
/// pairs for pages that yielded non-empty output.
|
||||
///
|
||||
/// Image extraction strategy: walk every page's resources looking for
|
||||
/// XObject streams with /Subtype /Image. Decode the raw stream bytes
|
||||
/// (lopdf handles FlateDecode/DCTDecode) and write them as-is to a
|
||||
/// temp file with the appropriate extension. Tesseract handles JPEG,
|
||||
/// PNG, TIFF, and BMP natively.
|
||||
fn ocr_pdf_images(
|
||||
doc: &lopdf::Document,
|
||||
source_filename: &str,
|
||||
) -> Result<Vec<(i32, String)>, String> {
|
||||
if !tesseract_available() {
|
||||
return Err(
|
||||
"tesseract not found on PATH — install with `apt install tesseract-ocr tesseract-ocr-eng`"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let pages = doc.get_pages();
|
||||
let mut results: Vec<(i32, String)> = Vec::new();
|
||||
|
||||
for (&page_num, &page_id) in &pages {
|
||||
let images = extract_page_images(doc, page_id);
|
||||
if images.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// OCR each image on this page and concatenate the text.
|
||||
let mut page_text = String::new();
|
||||
for (idx, img_bytes) in images.iter().enumerate() {
|
||||
match ocr_image_bytes(img_bytes, page_num, idx) {
|
||||
Ok(text) if !text.trim().is_empty() => {
|
||||
if !page_text.is_empty() { page_text.push('\n'); }
|
||||
page_text.push_str(text.trim());
|
||||
}
|
||||
Ok(_) => {} // empty OCR — skip
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"OCR failed on '{}' page {} image {}: {e}",
|
||||
source_filename, page_num, idx,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !page_text.is_empty() {
|
||||
results.push((page_num as i32, page_text));
|
||||
}
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Extract raw image bytes from a PDF page's XObject resources.
|
||||
/// Handles the common case: page /Resources → /XObject dict → streams
|
||||
/// with /Subtype /Image.
|
||||
fn extract_page_images(
|
||||
doc: &lopdf::Document,
|
||||
page_id: lopdf::ObjectId,
|
||||
) -> Vec<Vec<u8>> {
|
||||
let mut images = Vec::new();
|
||||
|
||||
let page_obj = match doc.get_object(page_id) {
|
||||
Ok(o) => o,
|
||||
Err(_) => return images,
|
||||
};
|
||||
let page_dict = match page_obj.as_dict() {
|
||||
Ok(d) => d,
|
||||
Err(_) => return images,
|
||||
};
|
||||
|
||||
// Resolve /Resources → /XObject
|
||||
let resources = page_dict
|
||||
.get(b"Resources")
|
||||
.ok()
|
||||
.and_then(|r| doc.dereference(r).ok())
|
||||
.map(|(_, o)| o.clone());
|
||||
let xobjects = resources
|
||||
.as_ref()
|
||||
.and_then(|r| r.as_dict().ok())
|
||||
.and_then(|d| d.get(b"XObject").ok())
|
||||
.and_then(|x| doc.dereference(x).ok())
|
||||
.map(|(_, o)| o.clone());
|
||||
|
||||
let xobj_dict = match xobjects.as_ref().and_then(|x| x.as_dict().ok()) {
|
||||
Some(d) => d,
|
||||
None => return images,
|
||||
};
|
||||
|
||||
for (_name, obj_ref) in xobj_dict.iter() {
|
||||
let obj = match doc.dereference(obj_ref) {
|
||||
Ok((_, o)) => o.clone(),
|
||||
Err(_) => continue,
|
||||
};
|
||||
let stream = match obj.as_stream() {
|
||||
Ok(s) => s,
|
||||
Err(_) => continue,
|
||||
};
|
||||
// Check /Subtype == /Image
|
||||
let is_image = stream.dict.get(b"Subtype")
|
||||
.ok()
|
||||
.and_then(|s| s.as_name().ok())
|
||||
.map(|n| n == b"Image")
|
||||
.unwrap_or(false);
|
||||
if !is_image {
|
||||
continue;
|
||||
}
|
||||
// Get decoded content — lopdf handles FlateDecode/DCTDecode.
|
||||
let content = match stream.decompressed_content() {
|
||||
Ok(c) => c,
|
||||
Err(_) => {
|
||||
// Try raw content as fallback (some DCT streams are stored raw).
|
||||
stream.content.clone()
|
||||
}
|
||||
};
|
||||
if !content.is_empty() {
|
||||
images.push(content);
|
||||
}
|
||||
}
|
||||
images
|
||||
}
|
||||
|
||||
/// Run Tesseract on raw image bytes. Writes to a temp file, shells out,
|
||||
/// reads the output. Returns the OCR'd text.
|
||||
fn ocr_image_bytes(
|
||||
img_bytes: &[u8],
|
||||
page_num: u32,
|
||||
img_idx: usize,
|
||||
) -> Result<String, String> {
|
||||
use std::io::Write;
|
||||
use std::process::Command;
|
||||
|
||||
let tmp_dir = std::env::temp_dir();
|
||||
// Detect image format from magic bytes to pick the right extension.
|
||||
let ext = if img_bytes.starts_with(&[0xFF, 0xD8]) {
|
||||
"jpg"
|
||||
} else if img_bytes.starts_with(&[0x89, 0x50, 0x4E, 0x47]) {
|
||||
"png"
|
||||
} else if img_bytes.starts_with(b"II") || img_bytes.starts_with(b"MM") {
|
||||
"tiff"
|
||||
} else {
|
||||
// Tesseract can often handle raw pixel data with the right options,
|
||||
// but writing as PNG is safer. For truly unknown formats, try PNG
|
||||
// wrapping; if Tesseract can't read it, it'll error cleanly.
|
||||
"png"
|
||||
};
|
||||
|
||||
let img_path = tmp_dir.join(format!("lh_ocr_p{page_num}_i{img_idx}.{ext}"));
|
||||
let out_base = tmp_dir.join(format!("lh_ocr_p{page_num}_i{img_idx}_out"));
|
||||
let out_path = tmp_dir.join(format!("lh_ocr_p{page_num}_i{img_idx}_out.txt"));
|
||||
|
||||
// Write image bytes to temp file.
|
||||
let mut f = std::fs::File::create(&img_path)
|
||||
.map_err(|e| format!("create temp image: {e}"))?;
|
||||
f.write_all(img_bytes)
|
||||
.map_err(|e| format!("write temp image: {e}"))?;
|
||||
drop(f);
|
||||
|
||||
// Run Tesseract.
|
||||
let status = Command::new("tesseract")
|
||||
.arg(&img_path)
|
||||
.arg(&out_base)
|
||||
.arg("--oem").arg("3") // LSTM + legacy combined
|
||||
.arg("--psm").arg("6") // assume uniform block of text
|
||||
.arg("-l").arg("eng")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map_err(|e| format!("tesseract exec: {e}"))?;
|
||||
|
||||
// Cleanup input regardless of outcome.
|
||||
let _ = std::fs::remove_file(&img_path);
|
||||
|
||||
if !status.success() {
|
||||
let _ = std::fs::remove_file(&out_path);
|
||||
return Err(format!("tesseract exited with status {status}"));
|
||||
}
|
||||
|
||||
let text = std::fs::read_to_string(&out_path).unwrap_or_default();
|
||||
let _ = std::fs::remove_file(&out_path);
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
/// Quick check: is `tesseract` on PATH?
|
||||
fn tesseract_available() -> bool {
|
||||
std::process::Command::new("tesseract")
|
||||
.arg("--version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
/// Streaming PostgreSQL ingest.
|
||||
///
|
||||
/// The original `db_ingest::import_postgres_table` loads every row into
|
||||
/// memory before emitting Parquet — fine for small tables, blows up on
|
||||
/// 1M+ rows. This module paginates via `ORDER BY <pk> LIMIT N OFFSET M`
|
||||
/// and streams batches into an `ArrowWriter`, closing once exhausted.
|
||||
///
|
||||
/// Pagination is OFFSET-based (not keyset) — simpler, works on any table
|
||||
/// without needing to know the PK type. OFFSET N scans N rows per fetch,
|
||||
/// which becomes quadratic for very large tables. Upgrade to keyset when
|
||||
/// a user actually ingests something that hurts.
|
||||
|
||||
use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use std::sync::Arc;
|
||||
use tokio_postgres::{Client, NoTls, types::Type as PgType};
|
||||
|
||||
use crate::db_ingest::DbConfig;
|
||||
|
||||
/// Request shape for streaming ingest — takes a DSN string and optional
|
||||
/// tuning knobs.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub struct PgStreamRequest {
|
||||
/// postgresql://user:pass@host:port/db — alternative to DbConfig.
|
||||
pub dsn: String,
|
||||
pub table: String,
|
||||
#[serde(default)]
|
||||
pub dataset_name: Option<String>,
|
||||
/// Rows per fetch. Default 10_000.
|
||||
#[serde(default)]
|
||||
pub batch_size: Option<usize>,
|
||||
/// Column to ORDER BY for stable pagination. If omitted, the first
|
||||
/// column returned by the schema probe is used.
|
||||
#[serde(default)]
|
||||
pub order_by: Option<String>,
|
||||
/// Hard cap on total rows (for sampling / previews).
|
||||
#[serde(default)]
|
||||
pub limit: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct PgStreamResult {
|
||||
pub table: String,
|
||||
pub rows: usize,
|
||||
pub batches: usize,
|
||||
pub columns: usize,
|
||||
pub schema: Vec<String>,
|
||||
pub parquet_bytes: u64,
|
||||
pub duration_secs: f32,
|
||||
}
|
||||
|
||||
/// Parse a postgresql:// DSN into a DbConfig.
|
||||
/// Supports: postgresql://[user[:password]@]host[:port][/db]
|
||||
/// Does NOT support: query parameters (sslmode etc) — add if needed.
|
||||
pub fn parse_dsn(dsn: &str) -> Result<DbConfig, String> {
|
||||
let rest = dsn
|
||||
.strip_prefix("postgresql://")
|
||||
.or_else(|| dsn.strip_prefix("postgres://"))
|
||||
.ok_or_else(|| "DSN must start with postgresql:// or postgres://".to_string())?;
|
||||
|
||||
// Split off path (database) first.
|
||||
let (auth_host, database) = match rest.split_once('/') {
|
||||
Some((ah, db)) => (ah, db.split('?').next().unwrap_or(db).to_string()),
|
||||
None => (rest, "postgres".to_string()),
|
||||
};
|
||||
|
||||
// Split user[:password] from host[:port].
|
||||
let (userpass, hostport) = match auth_host.rsplit_once('@') {
|
||||
Some((up, hp)) => (Some(up), hp),
|
||||
None => (None, auth_host),
|
||||
};
|
||||
|
||||
let (user, password) = match userpass {
|
||||
Some(up) => match up.split_once(':') {
|
||||
Some((u, p)) => (u.to_string(), p.to_string()),
|
||||
None => (up.to_string(), String::new()),
|
||||
},
|
||||
None => ("postgres".to_string(), String::new()),
|
||||
};
|
||||
|
||||
let (host, port) = match hostport.rsplit_once(':') {
|
||||
Some((h, p)) => (
|
||||
h.to_string(),
|
||||
p.parse::<u16>().map_err(|_| format!("invalid port in DSN: {p}"))?,
|
||||
),
|
||||
None => (hostport.to_string(), 5432),
|
||||
};
|
||||
|
||||
if host.is_empty() {
|
||||
return Err("DSN has no host".into());
|
||||
}
|
||||
|
||||
Ok(DbConfig { host, port, database, user, password })
|
||||
}
|
||||
|
||||
/// Stream a Postgres table as Parquet bytes.
|
||||
///
|
||||
/// Returns the full Parquet payload + summary stats. The payload is
|
||||
/// written to an in-memory buffer via `ArrowWriter` so each batch gets
|
||||
/// columnar compression as it arrives — memory footprint stays roughly
|
||||
/// at one batch plus Parquet footer state.
|
||||
pub async fn stream_table_to_parquet(
|
||||
req: &PgStreamRequest,
|
||||
) -> Result<(bytes::Bytes, PgStreamResult), String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
let config = parse_dsn(&req.dsn)?;
|
||||
let batch_size = req.batch_size.unwrap_or(10_000).max(1);
|
||||
|
||||
let (client, connection) = tokio_postgres::connect(&config.connection_string(), NoTls)
|
||||
.await
|
||||
.map_err(|e| format!("postgres connect: {e}"))?;
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
tracing::error!("pg connection: {e}");
|
||||
}
|
||||
});
|
||||
|
||||
// Probe schema.
|
||||
let columns = probe_columns(&client, &req.table).await?;
|
||||
if columns.is_empty() {
|
||||
return Err(format!("table '{}' not found or has no columns", req.table));
|
||||
}
|
||||
|
||||
let arrow_fields: Vec<Field> = columns
|
||||
.iter()
|
||||
.map(|(name, pg)| Field::new(name, pg_type_to_arrow(pg), true))
|
||||
.collect();
|
||||
let schema = Arc::new(Schema::new(arrow_fields));
|
||||
let schema_report: Vec<String> = columns
|
||||
.iter()
|
||||
.map(|(n, t)| format!("{}:{}", n, t))
|
||||
.collect();
|
||||
|
||||
// Pagination key: user-specified or first column.
|
||||
let order_col = req.order_by.clone().unwrap_or_else(|| columns[0].0.clone());
|
||||
|
||||
// Stream batches into ArrowWriter.
|
||||
let mut buf: Vec<u8> = Vec::with_capacity(1024 * 1024);
|
||||
let mut writer = ArrowWriter::try_new(&mut buf, schema.clone(), None)
|
||||
.map_err(|e| format!("ArrowWriter init: {e}"))?;
|
||||
|
||||
let mut total_rows: usize = 0;
|
||||
let mut batch_count: usize = 0;
|
||||
let row_cap = req.limit.unwrap_or(usize::MAX);
|
||||
|
||||
loop {
|
||||
let remaining = row_cap.saturating_sub(total_rows);
|
||||
if remaining == 0 { break; }
|
||||
let fetch = remaining.min(batch_size);
|
||||
|
||||
let sql = format!(
|
||||
"SELECT * FROM \"{}\" ORDER BY \"{}\" LIMIT {} OFFSET {}",
|
||||
req.table, order_col, fetch, total_rows,
|
||||
);
|
||||
|
||||
let rows = client
|
||||
.query(&sql, &[])
|
||||
.await
|
||||
.map_err(|e| format!("fetch batch at offset {total_rows}: {e}"))?;
|
||||
|
||||
if rows.is_empty() { break; }
|
||||
|
||||
let n = rows.len();
|
||||
let arrays: Vec<ArrayRef> = columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, (_, pg))| rows_to_column(&rows, idx, pg))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch: {e}"))?;
|
||||
writer.write(&batch).map_err(|e| format!("ArrowWriter::write: {e}"))?;
|
||||
|
||||
total_rows += n;
|
||||
batch_count += 1;
|
||||
tracing::info!(
|
||||
"pg stream '{}': fetched batch {} ({} rows, total {})",
|
||||
req.table, batch_count, n, total_rows,
|
||||
);
|
||||
|
||||
if n < fetch { break; } // short read = end of table
|
||||
}
|
||||
|
||||
writer.close().map_err(|e| format!("ArrowWriter::close: {e}"))?;
|
||||
let parquet_bytes = buf.len() as u64;
|
||||
let duration = t0.elapsed().as_secs_f32();
|
||||
|
||||
let result = PgStreamResult {
|
||||
table: req.table.clone(),
|
||||
rows: total_rows,
|
||||
batches: batch_count,
|
||||
columns: columns.len(),
|
||||
schema: schema_report,
|
||||
parquet_bytes,
|
||||
duration_secs: duration,
|
||||
};
|
||||
|
||||
Ok((bytes::Bytes::from(buf), result))
|
||||
}
|
||||
|
||||
async fn probe_columns(client: &Client, table: &str) -> Result<Vec<(String, PgType)>, String> {
|
||||
let stmt = client
|
||||
.prepare(&format!("SELECT * FROM \"{}\" LIMIT 0", table))
|
||||
.await
|
||||
.map_err(|e| format!("prepare schema probe: {e}"))?;
|
||||
Ok(stmt
|
||||
.columns()
|
||||
.iter()
|
||||
.map(|c| (c.name().to_string(), c.type_().clone()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn pg_type_to_arrow(pg: &PgType) -> DataType {
|
||||
match *pg {
|
||||
PgType::BOOL => DataType::Boolean,
|
||||
PgType::INT2 | PgType::INT4 => DataType::Int32,
|
||||
PgType::INT8 | PgType::OID => DataType::Int64,
|
||||
PgType::FLOAT4 | PgType::FLOAT8 | PgType::NUMERIC => DataType::Float64,
|
||||
_ => DataType::Utf8,
|
||||
}
|
||||
}
|
||||
|
||||
fn rows_to_column(
|
||||
rows: &[tokio_postgres::Row],
|
||||
idx: usize,
|
||||
pg: &PgType,
|
||||
) -> Result<ArrayRef, String> {
|
||||
match *pg {
|
||||
PgType::BOOL => {
|
||||
let v: Vec<Option<bool>> = rows.iter().map(|r| r.try_get(idx).ok()).collect();
|
||||
Ok(Arc::new(BooleanArray::from(v)))
|
||||
}
|
||||
PgType::INT2 => {
|
||||
let v: Vec<Option<i32>> = rows.iter()
|
||||
.map(|r| r.try_get::<_, i16>(idx).ok().map(|x| x as i32)).collect();
|
||||
Ok(Arc::new(Int32Array::from(v)))
|
||||
}
|
||||
PgType::INT4 => {
|
||||
let v: Vec<Option<i32>> = rows.iter().map(|r| r.try_get(idx).ok()).collect();
|
||||
Ok(Arc::new(Int32Array::from(v)))
|
||||
}
|
||||
PgType::INT8 | PgType::OID => {
|
||||
let v: Vec<Option<i64>> = rows.iter().map(|r| r.try_get(idx).ok()).collect();
|
||||
Ok(Arc::new(Int64Array::from(v)))
|
||||
}
|
||||
PgType::FLOAT4 => {
|
||||
let v: Vec<Option<f64>> = rows.iter()
|
||||
.map(|r| r.try_get::<_, f32>(idx).ok().map(|x| x as f64)).collect();
|
||||
Ok(Arc::new(Float64Array::from(v)))
|
||||
}
|
||||
PgType::FLOAT8 => {
|
||||
let v: Vec<Option<f64>> = rows.iter().map(|r| r.try_get(idx).ok()).collect();
|
||||
Ok(Arc::new(Float64Array::from(v)))
|
||||
}
|
||||
_ => {
|
||||
// Safe-default per ADR-010: everything else becomes string.
|
||||
let v: Vec<Option<String>> = rows.iter().map(|r| {
|
||||
r.try_get::<_, String>(idx).ok()
|
||||
.or_else(|| r.try_get::<_, serde_json::Value>(idx).ok().map(|j| j.to_string()))
|
||||
.or_else(|| {
|
||||
// Timestamps + uuid, rendered via Display.
|
||||
r.try_get::<_, chrono::DateTime<chrono::Utc>>(idx).ok()
|
||||
.map(|t| t.to_rfc3339())
|
||||
})
|
||||
.or_else(|| r.try_get::<_, uuid::Uuid>(idx).ok().map(|u| u.to_string()))
|
||||
.or(Some(String::new()))
|
||||
}).collect();
|
||||
Ok(Arc::new(StringArray::from(v)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_full() {
|
||||
let c = parse_dsn("postgresql://daisy:secret@db.example.com:6543/my_db").unwrap();
|
||||
assert_eq!(c.host, "db.example.com");
|
||||
assert_eq!(c.port, 6543);
|
||||
assert_eq!(c.user, "daisy");
|
||||
assert_eq!(c.password, "secret");
|
||||
assert_eq!(c.database, "my_db");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_minimal() {
|
||||
let c = parse_dsn("postgresql://localhost/knowledge_base").unwrap();
|
||||
assert_eq!(c.host, "localhost");
|
||||
assert_eq!(c.port, 5432);
|
||||
assert_eq!(c.user, "postgres");
|
||||
assert_eq!(c.password, "");
|
||||
assert_eq!(c.database, "knowledge_base");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_no_password() {
|
||||
let c = parse_dsn("postgres://postgres@127.0.0.1:5432/mydb").unwrap();
|
||||
assert_eq!(c.user, "postgres");
|
||||
assert_eq!(c.password, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_dsn_rejects_non_pg() {
|
||||
assert!(parse_dsn("http://host/db").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
use arrow::datatypes::SchemaRef;
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use bytes::Bytes;
|
||||
use object_store::ObjectStore;
|
||||
use std::sync::Arc;
|
||||
|
||||
use catalogd::registry::Registry;
|
||||
use shared::arrow_helpers::{fingerprint_schema, record_batch_to_parquet};
|
||||
use shared::types::{ObjectRef, SchemaFingerprint};
|
||||
use storaged::ops;
|
||||
|
||||
use crate::detect::{FileType, content_hash, detect_file_type};
|
||||
use crate::csv_ingest;
|
||||
use crate::json_ingest;
|
||||
use crate::pdf_ingest;
|
||||
|
||||
/// Result of an ingest operation.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct IngestResult {
|
||||
pub dataset_name: String,
|
||||
pub file_type: String,
|
||||
pub rows: usize,
|
||||
pub columns: usize,
|
||||
pub storage_key: String,
|
||||
pub content_hash: String,
|
||||
pub schema_fingerprint: String,
|
||||
pub deduplicated: bool,
|
||||
}
|
||||
|
||||
/// Full ingest pipeline: detect → parse → dedup → store → register.
|
||||
///
|
||||
/// Bucket name (federation layer 2) is recorded on the ObjectRef so the
|
||||
/// catalog knows which bucket holds the data — defaults to "primary" via
|
||||
/// the ingest service's resolve_bucket helper.
|
||||
pub async fn ingest_file(
|
||||
filename: &str,
|
||||
content: &[u8],
|
||||
dataset_name: Option<&str>,
|
||||
store: &Arc<dyn ObjectStore>,
|
||||
registry: &Registry,
|
||||
) -> Result<IngestResult, String> {
|
||||
ingest_file_to_bucket(filename, content, dataset_name, "primary", store, registry).await
|
||||
}
|
||||
|
||||
/// Same as `ingest_file` but with explicit target bucket name on the
|
||||
/// ObjectRef. Header-aware ingest endpoints call this directly.
|
||||
pub async fn ingest_file_to_bucket(
|
||||
filename: &str,
|
||||
content: &[u8],
|
||||
dataset_name: Option<&str>,
|
||||
bucket: &str,
|
||||
store: &Arc<dyn ObjectStore>,
|
||||
registry: &Registry,
|
||||
) -> Result<IngestResult, String> {
|
||||
// 1. Detect file type
|
||||
let file_type = detect_file_type(filename, content);
|
||||
tracing::info!("ingesting '{}' as {:?} ({} bytes)", filename, file_type, content.len());
|
||||
|
||||
// 2. Parse into Arrow
|
||||
let (schema, batches) = match file_type {
|
||||
FileType::Csv => csv_ingest::parse_csv(content)?,
|
||||
FileType::Json | FileType::NdJson => json_ingest::parse_json(content)?,
|
||||
FileType::Pdf => pdf_ingest::parse_pdf(content, filename)?,
|
||||
FileType::Text => parse_text_file(content, filename)?,
|
||||
FileType::Unknown => return Err(format!("unknown file type for '{filename}'")),
|
||||
};
|
||||
|
||||
let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
|
||||
let total_cols = schema.fields().len();
|
||||
|
||||
if total_rows == 0 {
|
||||
return Err("file contains no data".into());
|
||||
}
|
||||
|
||||
// 3. Content hash for dedup
|
||||
let hash = content_hash(content);
|
||||
|
||||
// 4. Check if already ingested (same content hash)
|
||||
let existing = registry.list().await;
|
||||
if existing.iter().any(|d| d.schema_fingerprint.0 == hash) {
|
||||
tracing::info!("file already ingested (hash: {}), skipping", &hash[..12]);
|
||||
return Ok(IngestResult {
|
||||
dataset_name: dataset_name.unwrap_or(filename).to_string(),
|
||||
file_type: format!("{:?}", file_type),
|
||||
rows: total_rows,
|
||||
columns: total_cols,
|
||||
storage_key: String::new(),
|
||||
content_hash: hash,
|
||||
schema_fingerprint: String::new(),
|
||||
deduplicated: true,
|
||||
});
|
||||
}
|
||||
|
||||
// 5. Convert to Parquet
|
||||
let mut all_parquet = Vec::new();
|
||||
for batch in &batches {
|
||||
let pq = record_batch_to_parquet(batch)?;
|
||||
all_parquet.extend_from_slice(&pq);
|
||||
}
|
||||
let parquet_bytes = Bytes::from(all_parquet);
|
||||
let parquet_size = parquet_bytes.len() as u64;
|
||||
|
||||
// 6. Store in object storage
|
||||
let name = dataset_name.unwrap_or_else(|| {
|
||||
filename.rsplit('/').next().unwrap_or(filename)
|
||||
.rsplit('.').last().unwrap_or(filename)
|
||||
});
|
||||
let safe_name = sanitize_dataset_name(name);
|
||||
let storage_key = format!("datasets/{}.parquet", safe_name);
|
||||
|
||||
ops::put(store, &storage_key, parquet_bytes).await?;
|
||||
tracing::info!("stored {} as {} ({} bytes)", filename, storage_key, parquet_size);
|
||||
|
||||
// 7. Register in catalog with rich metadata
|
||||
let schema_fp = fingerprint_schema(&schema);
|
||||
let now = chrono::Utc::now();
|
||||
let obj_ref = ObjectRef {
|
||||
bucket: bucket.to_string(),
|
||||
key: storage_key.clone(),
|
||||
size_bytes: parquet_size,
|
||||
created_at: now,
|
||||
};
|
||||
|
||||
// Register base dataset
|
||||
registry.register(
|
||||
safe_name.clone(),
|
||||
SchemaFingerprint(hash.clone()),
|
||||
vec![obj_ref],
|
||||
).await?;
|
||||
|
||||
// Auto-populate metadata: PII detection, column info, lineage, row count
|
||||
let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
|
||||
let dataset_sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
|
||||
|
||||
let column_meta: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
|
||||
let sens = shared::pii::detect_sensitivity(f.name());
|
||||
shared::types::ColumnMeta {
|
||||
name: f.name().clone(),
|
||||
data_type: f.data_type().to_string(),
|
||||
sensitivity: sens.clone(),
|
||||
description: String::new(),
|
||||
is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let lineage = shared::types::Lineage {
|
||||
source_system: format!("{:?}", file_type).to_lowercase(),
|
||||
source_file: filename.to_string(),
|
||||
ingest_job: format!("ingest-{}", now.timestamp_millis()),
|
||||
ingest_timestamp: now,
|
||||
parent_datasets: vec![],
|
||||
};
|
||||
|
||||
let pii_count = column_meta.iter().filter(|c| c.is_pii).count();
|
||||
if pii_count > 0 {
|
||||
tracing::info!("auto-detected {} PII columns in '{}'", pii_count, safe_name);
|
||||
}
|
||||
|
||||
let _ = registry.update_metadata(&safe_name, catalogd::registry::MetadataUpdate {
|
||||
sensitivity: dataset_sensitivity,
|
||||
columns: Some(column_meta),
|
||||
lineage: Some(lineage),
|
||||
row_count: Some(total_rows as u64),
|
||||
..Default::default()
|
||||
}).await;
|
||||
|
||||
// Phase C: if this dataset already had embeddings, they're now stale.
|
||||
// mark_embeddings_stale is a no-op for never-embedded datasets.
|
||||
let _ = registry.mark_embeddings_stale(&safe_name).await;
|
||||
|
||||
Ok(IngestResult {
|
||||
dataset_name: safe_name,
|
||||
file_type: format!("{:?}", file_type),
|
||||
rows: total_rows,
|
||||
columns: total_cols,
|
||||
storage_key,
|
||||
content_hash: hash,
|
||||
schema_fingerprint: schema_fp.0,
|
||||
deduplicated: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse plain text / SMS logs into a simple table.
|
||||
fn parse_text_file(content: &[u8], filename: &str) -> Result<(SchemaRef, Vec<RecordBatch>), String> {
|
||||
let text = String::from_utf8_lossy(content);
|
||||
let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
|
||||
|
||||
if lines.is_empty() {
|
||||
return Err("text file is empty".into());
|
||||
}
|
||||
|
||||
let schema = Arc::new(arrow::datatypes::Schema::new(vec![
|
||||
arrow::datatypes::Field::new("source_file", arrow::datatypes::DataType::Utf8, false),
|
||||
arrow::datatypes::Field::new("line_number", arrow::datatypes::DataType::Int32, false),
|
||||
arrow::datatypes::Field::new("text", arrow::datatypes::DataType::Utf8, false),
|
||||
]));
|
||||
|
||||
let sources: Vec<&str> = vec![filename; lines.len()];
|
||||
let line_nums: Vec<i32> = (1..=lines.len() as i32).collect();
|
||||
|
||||
let arrays: Vec<arrow::array::ArrayRef> = vec![
|
||||
Arc::new(arrow::array::StringArray::from(sources)),
|
||||
Arc::new(arrow::array::Int32Array::from(line_nums)),
|
||||
Arc::new(arrow::array::StringArray::from(lines)),
|
||||
];
|
||||
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
Ok((schema, vec![batch]))
|
||||
}
|
||||
|
||||
fn sanitize_dataset_name(name: &str) -> String {
|
||||
let clean: String = name.chars()
|
||||
.map(|c| if c.is_alphanumeric() || c == '_' { c.to_ascii_lowercase() } else { '_' })
|
||||
.collect();
|
||||
let trimmed = clean.trim_matches('_').to_string();
|
||||
if trimmed.is_empty() { "unnamed_dataset".to_string() } else { trimmed }
|
||||
}
|
||||
@@ -0,0 +1,725 @@
|
||||
//! Scheduled ingest — the production-substrate piece that turns the
|
||||
//! lakehouse from "manual API toolkit" into a system that runs itself.
|
||||
//!
|
||||
//! A `ScheduleDef` declares "what to ingest" + "when to fire." A
|
||||
//! background `Scheduler` task wakes periodically, asks each enabled
|
||||
//! schedule whether it's due, and runs the matching ingest path. Outcomes
|
||||
//! land back on the schedule so operators can see what happened without
|
||||
//! `journalctl` archaeology.
|
||||
//!
|
||||
//! Storage shape: one JSON file per schedule at
|
||||
//! `primary://_schedules/{id}.json`. Serializable via serde, rebuilt on
|
||||
//! startup. Same write-once shape as the rest of the catalog — no DB.
|
||||
//!
|
||||
//! Concurrency model: single-flight per schedule_id. The scheduler holds
|
||||
//! a `HashSet<String>` of currently-running ids; a tick that finds a
|
||||
//! schedule already in flight skips it (no queueing — a slow ingest
|
||||
//! shouldn't pile up runs). Different schedules run concurrently.
|
||||
//!
|
||||
//! What's deliberately not in scope here:
|
||||
//! - Sub-minute cron precision. 5-field Unix cron is supported; seconds
|
||||
//! are pinned to 0. Intervals cover sub-minute cases; cron is
|
||||
//! easy to bolt on later.
|
||||
//! - Backoff / retry policies. A failed run records the failure and
|
||||
//! schedules `next_run_at` as if it succeeded — no exponential backoff.
|
||||
//! - Distributed coordination. Single-machine scheduler; no leader
|
||||
//! election. If you run two gateway instances they'll both fire.
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use object_store::ObjectStore;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use storaged::ops;
|
||||
|
||||
const PREFIX: &str = "_schedules";
|
||||
|
||||
// =================== Public types ===================
|
||||
|
||||
/// What kind of ingest the schedule fires.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ScheduleKind {
|
||||
/// Pull a MySQL/MariaDB table via the my_stream path.
|
||||
Mysql {
|
||||
/// `mysql://user:pass@host:port/db` — DSN with full credentials.
|
||||
/// Stored encrypted-at-rest is a future concern; for now it lives
|
||||
/// in plain JSON in the catalog. Don't put production root creds
|
||||
/// here without disk-level encryption.
|
||||
dsn: String,
|
||||
table: String,
|
||||
#[serde(default)]
|
||||
dataset_name: Option<String>,
|
||||
#[serde(default)]
|
||||
batch_size: Option<usize>,
|
||||
#[serde(default)]
|
||||
order_by: Option<String>,
|
||||
},
|
||||
/// Pull a Postgres table via the pg_stream path.
|
||||
Postgres {
|
||||
dsn: String,
|
||||
table: String,
|
||||
#[serde(default)]
|
||||
dataset_name: Option<String>,
|
||||
#[serde(default)]
|
||||
batch_size: Option<usize>,
|
||||
#[serde(default)]
|
||||
order_by: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ScheduleKind {
|
||||
pub fn label(&self) -> String {
|
||||
match self {
|
||||
ScheduleKind::Mysql { table, .. } => format!("mysql:{table}"),
|
||||
ScheduleKind::Postgres { table, .. } => format!("postgres:{table}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// When the schedule fires.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ScheduleTrigger {
|
||||
/// Run every N seconds. Time anchor = compute_next_run_at decision.
|
||||
Interval { secs: u64 },
|
||||
/// 5-field Unix cron expression: "min hour dom month dow".
|
||||
/// Seconds are implicitly 0 (minute-granularity scheduling).
|
||||
/// Examples: "15 14 * * *" = 14:15 UTC daily, "0 */6 * * *" = every
|
||||
/// 6 hours on the hour, "0 9 * * 1-5" = 09:00 UTC weekdays.
|
||||
/// Timezone: UTC always — the scheduler's clock is UTC and we don't
|
||||
/// carry a per-schedule timezone field.
|
||||
Cron { expr: String },
|
||||
}
|
||||
|
||||
/// Outcome of a single run, kept on the schedule for observability.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RunOutcome {
|
||||
pub at: DateTime<Utc>,
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
pub rows: Option<usize>,
|
||||
pub duration_secs: f32,
|
||||
}
|
||||
|
||||
/// One scheduled ingest definition.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ScheduleDef {
|
||||
pub id: String,
|
||||
pub kind: ScheduleKind,
|
||||
pub trigger: ScheduleTrigger,
|
||||
pub enabled: bool,
|
||||
pub created_at: DateTime<Utc>,
|
||||
#[serde(default)]
|
||||
pub created_by: String,
|
||||
/// When the next run is due. Recomputed after each completion via
|
||||
/// `compute_next_run_at`. On creation: set to "now" so the schedule
|
||||
/// fires on the next scheduler tick.
|
||||
pub next_run_at: DateTime<Utc>,
|
||||
#[serde(default)]
|
||||
pub last_run_at: Option<DateTime<Utc>>,
|
||||
#[serde(default)]
|
||||
pub last_outcome: Option<RunOutcome>,
|
||||
/// Total successful runs and total failures since creation. Useful
|
||||
/// for at-a-glance "is this schedule healthy?" checks.
|
||||
#[serde(default)]
|
||||
pub run_count: u64,
|
||||
#[serde(default)]
|
||||
pub failure_count: u64,
|
||||
}
|
||||
|
||||
impl ScheduleDef {
|
||||
/// True if this schedule should fire at `now`. Honors `enabled`.
|
||||
pub fn is_due(&self, now: DateTime<Utc>) -> bool {
|
||||
self.enabled && now >= self.next_run_at
|
||||
}
|
||||
}
|
||||
|
||||
// =================== Trigger semantics ===================
|
||||
//
|
||||
// J: THIS IS YOURS TO IMPLEMENT
|
||||
// =====================================================
|
||||
//
|
||||
// `compute_next_run_at` decides when a schedule fires next. The current
|
||||
// state is the "since last completion" semantics — simple but has real
|
||||
// trade-offs vs other strategies:
|
||||
//
|
||||
// 1. SINCE LAST COMPLETION (current default below):
|
||||
// next = completed_at + interval
|
||||
// Easy to reason about. A long-running ingest delays the next fire
|
||||
// by however long it took. Drift is bounded — the schedule never
|
||||
// "falls behind" trying to catch up.
|
||||
//
|
||||
// 2. ANCHORED INTERVAL:
|
||||
// next = previous_target + interval, regardless of when run finished
|
||||
// Keeps wall-clock alignment ("every 30min on the :00 and :30").
|
||||
// Risk: if ingest takes longer than the interval, the next run is
|
||||
// "due immediately" and you can fire back-to-back forever.
|
||||
//
|
||||
// 3. ANCHORED + SKIP:
|
||||
// Anchored, but skip a tick if the previous one is still running.
|
||||
// Best of both, slightly more code.
|
||||
//
|
||||
// 4. CRON:
|
||||
// Parse the expression, find the next match after `now`. Most
|
||||
// expressive but pulls in a cron parser dep.
|
||||
//
|
||||
pub fn compute_next_run_at(
|
||||
trigger: &ScheduleTrigger,
|
||||
completed_at: DateTime<Utc>,
|
||||
_previous_next: DateTime<Utc>,
|
||||
) -> DateTime<Utc> {
|
||||
match trigger {
|
||||
ScheduleTrigger::Interval { secs } => {
|
||||
completed_at + Duration::seconds(*secs as i64)
|
||||
}
|
||||
ScheduleTrigger::Cron { expr } => {
|
||||
next_cron_fire(expr, completed_at)
|
||||
// Creation-time validation (see `validate_trigger`) rejects
|
||||
// unparseable expressions, so this fallback is only reached
|
||||
// if the schedule JSON was hand-edited on disk. One-hour
|
||||
// window keeps a bad expr from fire-looping the system.
|
||||
.unwrap_or(completed_at + Duration::hours(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a Vixie/POSIX cron expression and return the next fire time
|
||||
/// strictly after `after`. Accepts 5-field (`min hour dom month dow`) and
|
||||
/// 6-field (with leading `sec`) forms natively — croner handles both.
|
||||
/// Day-of-week follows Unix convention: 0=Sun, 1=Mon, …, 6=Sat (7=Sun as
|
||||
/// synonym). Returns None if the expression fails to parse or produces no
|
||||
/// upcoming match (which can happen for year-bounded patterns in the past).
|
||||
fn next_cron_fire(expr: &str, after: DateTime<Utc>) -> Option<DateTime<Utc>> {
|
||||
use std::str::FromStr;
|
||||
let cron = croner::Cron::from_str(expr).ok()?;
|
||||
cron.find_next_occurrence(&after, false).ok()
|
||||
}
|
||||
|
||||
/// Reject a ScheduleTrigger whose content can't be interpreted — used by
|
||||
/// the HTTP handlers to fail fast at create/patch time rather than falling
|
||||
/// back silently at fire time. Interval triggers get a sanity gate
|
||||
/// (no zero secs); cron triggers get a full parse through croner.
|
||||
pub fn validate_trigger(trigger: &ScheduleTrigger) -> Result<(), String> {
|
||||
match trigger {
|
||||
ScheduleTrigger::Interval { secs } => {
|
||||
if *secs == 0 {
|
||||
return Err("interval secs must be > 0".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
ScheduleTrigger::Cron { expr } => {
|
||||
use std::str::FromStr;
|
||||
croner::Cron::from_str(expr)
|
||||
.map_err(|e| format!("invalid cron expression '{expr}': {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =================== Persistence ===================
|
||||
|
||||
/// Storage path for a schedule's JSON record.
|
||||
fn schedule_key(id: &str) -> String {
|
||||
let safe: String = id
|
||||
.chars()
|
||||
.map(|c| if c.is_ascii_alphanumeric() || c == '_' || c == '-' { c } else { '_' })
|
||||
.collect();
|
||||
format!("{PREFIX}/{safe}.json")
|
||||
}
|
||||
|
||||
/// In-memory + on-disk schedule registry. The Scheduler reads from here;
|
||||
/// HTTP CRUD writes through here.
|
||||
#[derive(Clone)]
|
||||
pub struct ScheduleStore {
|
||||
/// Bucket where schedule JSON lives. Always primary today — a
|
||||
/// future enhancement could move to per-profile schedule buckets.
|
||||
store: Arc<dyn ObjectStore>,
|
||||
schedules: Arc<RwLock<HashMap<String, ScheduleDef>>>,
|
||||
}
|
||||
|
||||
impl ScheduleStore {
|
||||
pub fn new(store: Arc<dyn ObjectStore>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
schedules: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Load every persisted schedule into memory. Call on startup.
|
||||
pub async fn rebuild(&self) -> Result<usize, String> {
|
||||
let keys = ops::list(&self.store, Some(&format!("{PREFIX}/")))
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let mut count = 0usize;
|
||||
for key in keys {
|
||||
if !key.ends_with(".json") { continue; }
|
||||
match ops::get(&self.store, &key).await {
|
||||
Ok(bytes) => match serde_json::from_slice::<ScheduleDef>(&bytes) {
|
||||
Ok(def) => {
|
||||
self.schedules.write().await.insert(def.id.clone(), def);
|
||||
count += 1;
|
||||
}
|
||||
Err(e) => tracing::warn!("schedule {key}: malformed: {e}"),
|
||||
},
|
||||
Err(e) => tracing::warn!("schedule {key}: read failed: {e}"),
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub async fn list(&self) -> Vec<ScheduleDef> {
|
||||
let mut out: Vec<ScheduleDef> = self.schedules.read().await.values().cloned().collect();
|
||||
out.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
out
|
||||
}
|
||||
|
||||
pub async fn get(&self, id: &str) -> Option<ScheduleDef> {
|
||||
self.schedules.read().await.get(id).cloned()
|
||||
}
|
||||
|
||||
pub async fn put(&self, def: ScheduleDef) -> Result<ScheduleDef, String> {
|
||||
let json = serde_json::to_vec_pretty(&def).map_err(|e| e.to_string())?;
|
||||
ops::put(&self.store, &schedule_key(&def.id), json.into()).await?;
|
||||
self.schedules.write().await.insert(def.id.clone(), def.clone());
|
||||
Ok(def)
|
||||
}
|
||||
|
||||
pub async fn delete(&self, id: &str) -> Result<bool, String> {
|
||||
// Object_store's delete is idempotent — silently OKs missing keys.
|
||||
let _ = ops::delete(&self.store, &schedule_key(id)).await;
|
||||
Ok(self.schedules.write().await.remove(id).is_some())
|
||||
}
|
||||
|
||||
/// Update completion metadata + persist. Used by the scheduler
|
||||
/// after a run finishes (success or failure).
|
||||
pub async fn record_run(
|
||||
&self,
|
||||
id: &str,
|
||||
outcome: RunOutcome,
|
||||
) -> Result<(), String> {
|
||||
let mut guard = self.schedules.write().await;
|
||||
let Some(def) = guard.get_mut(id) else {
|
||||
return Err(format!("schedule '{id}' not registered"));
|
||||
};
|
||||
def.last_run_at = Some(outcome.at);
|
||||
if outcome.success {
|
||||
def.run_count += 1;
|
||||
} else {
|
||||
def.failure_count += 1;
|
||||
}
|
||||
def.next_run_at = compute_next_run_at(&def.trigger, outcome.at, def.next_run_at);
|
||||
def.last_outcome = Some(outcome);
|
||||
|
||||
let json = serde_json::to_vec_pretty(def).map_err(|e| e.to_string())?;
|
||||
let key = schedule_key(id);
|
||||
let store = self.store.clone();
|
||||
// Drop the write lock before async I/O.
|
||||
drop(guard);
|
||||
ops::put(&store, &key, json.into()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// =================== Scheduler task ===================
|
||||
|
||||
/// Long-running task that fires due schedules. Spawned at gateway
|
||||
/// startup. Stop semantics: drop the handle (the loop exits when its
|
||||
/// store is the only reference left, or sooner if you wire a stop signal).
|
||||
pub struct Scheduler {
|
||||
pub store: ScheduleStore,
|
||||
pub ingest: SchedulerIngestDeps,
|
||||
/// Polling cadence for the loop itself. The schedules' own intervals
|
||||
/// are independent — this just controls how granularly we check
|
||||
/// "is anything due?" Default 10s.
|
||||
pub tick_secs: u64,
|
||||
}
|
||||
|
||||
/// What the scheduler needs to actually run an ingest. We pass in only
|
||||
/// what's needed — keeps the dep set narrow.
|
||||
#[derive(Clone)]
|
||||
pub struct SchedulerIngestDeps {
|
||||
pub store: Arc<dyn ObjectStore>,
|
||||
pub registry: catalogd::registry::Registry,
|
||||
pub buckets: Arc<storaged::registry::BucketRegistry>,
|
||||
/// Optional Phase 16.5 hook — push a DatasetAppended event if any
|
||||
/// HNSW index is bound to the dataset that just got refreshed.
|
||||
pub agent_handle: Option<vectord::agent::AgentHandle>,
|
||||
pub index_registry: vectord::index_registry::IndexRegistry,
|
||||
}
|
||||
|
||||
impl Scheduler {
|
||||
/// Spawn the loop in the background. Returns immediately.
|
||||
pub fn spawn(self) {
|
||||
tokio::spawn(async move {
|
||||
self.run().await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn run(self) {
|
||||
tracing::info!("scheduler started — tick={}s", self.tick_secs);
|
||||
let in_flight: Arc<RwLock<HashSet<String>>> = Arc::new(RwLock::new(HashSet::new()));
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(self.tick_secs)).await;
|
||||
let now = Utc::now();
|
||||
let due: Vec<ScheduleDef> = self.store.list().await
|
||||
.into_iter()
|
||||
.filter(|s| s.is_due(now))
|
||||
.collect();
|
||||
if due.is_empty() { continue; }
|
||||
|
||||
for def in due {
|
||||
{
|
||||
let in_flight_guard = in_flight.read().await;
|
||||
if in_flight_guard.contains(&def.id) {
|
||||
// Previous run still going — skip this tick.
|
||||
// We don't reschedule; next tick re-evaluates.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let store = self.store.clone();
|
||||
let ingest = self.ingest.clone();
|
||||
let in_flight = in_flight.clone();
|
||||
let id = def.id.clone();
|
||||
|
||||
in_flight.write().await.insert(id.clone());
|
||||
|
||||
tokio::spawn(async move {
|
||||
let t0 = std::time::Instant::now();
|
||||
let outcome = run_schedule(&def, &ingest).await;
|
||||
let elapsed = t0.elapsed().as_secs_f32();
|
||||
let outcome = RunOutcome {
|
||||
at: Utc::now(),
|
||||
success: outcome.is_ok(),
|
||||
message: match &outcome {
|
||||
Ok(s) => s.clone(),
|
||||
Err(e) => e.clone(),
|
||||
},
|
||||
rows: outcome.as_ref().ok().and_then(parse_rows_from_message),
|
||||
duration_secs: elapsed,
|
||||
};
|
||||
if let Err(e) = store.record_run(&id, outcome).await {
|
||||
tracing::warn!("scheduler: record_run({id}): {e}");
|
||||
}
|
||||
in_flight.write().await.remove(&id);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pluck a "rows=N" hint out of a success message — best-effort, no
|
||||
/// hard contract. Lets us surface row counts in the schedule status
|
||||
/// endpoint without forcing every ingest path to return structured
|
||||
/// stats here.
|
||||
fn parse_rows_from_message(msg: &String) -> Option<usize> {
|
||||
msg.split_whitespace()
|
||||
.find_map(|tok| tok.strip_prefix("rows=").and_then(|n| n.parse().ok()))
|
||||
}
|
||||
|
||||
/// Fire one schedule's ingest. Returns Ok(success_message) or Err(reason).
|
||||
pub async fn run_schedule(
|
||||
def: &ScheduleDef,
|
||||
deps: &SchedulerIngestDeps,
|
||||
) -> Result<String, String> {
|
||||
tracing::info!("scheduler: firing '{}' ({})", def.id, def.kind.label());
|
||||
match &def.kind {
|
||||
ScheduleKind::Mysql { dsn, table, dataset_name, batch_size, order_by } => {
|
||||
let req = crate::my_stream::MyStreamRequest {
|
||||
dsn: dsn.clone(),
|
||||
table: table.clone(),
|
||||
dataset_name: dataset_name.clone(),
|
||||
batch_size: *batch_size,
|
||||
order_by: order_by.clone(),
|
||||
limit: None,
|
||||
};
|
||||
run_mysql(req, deps).await
|
||||
}
|
||||
ScheduleKind::Postgres { dsn, table, dataset_name, batch_size, order_by } => {
|
||||
let req = crate::pg_stream::PgStreamRequest {
|
||||
dsn: dsn.clone(),
|
||||
table: table.clone(),
|
||||
dataset_name: dataset_name.clone(),
|
||||
batch_size: *batch_size,
|
||||
order_by: order_by.clone(),
|
||||
limit: None,
|
||||
};
|
||||
run_postgres(req, deps).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_mysql(
|
||||
req: crate::my_stream::MyStreamRequest,
|
||||
deps: &SchedulerIngestDeps,
|
||||
) -> Result<String, String> {
|
||||
let (parquet, result) = crate::my_stream::stream_table_to_parquet(&req).await?;
|
||||
if result.rows == 0 {
|
||||
return Ok(format!("rows=0 — table empty"));
|
||||
}
|
||||
persist_and_register(
|
||||
&parquet, &result.table, req.dataset_name.as_deref(),
|
||||
result.rows, "mysql", &req.dsn, deps,
|
||||
).await
|
||||
}
|
||||
|
||||
async fn run_postgres(
|
||||
req: crate::pg_stream::PgStreamRequest,
|
||||
deps: &SchedulerIngestDeps,
|
||||
) -> Result<String, String> {
|
||||
let (parquet, result) = crate::pg_stream::stream_table_to_parquet(&req).await?;
|
||||
if result.rows == 0 {
|
||||
return Ok(format!("rows=0 — table empty"));
|
||||
}
|
||||
persist_and_register(
|
||||
&parquet, &result.table, req.dataset_name.as_deref(),
|
||||
result.rows, "postgresql", &req.dsn, deps,
|
||||
).await
|
||||
}
|
||||
|
||||
/// Shared write+register path. Mirrors the HTTP ingest handlers in
|
||||
/// service.rs so the scheduled path produces identical catalog output:
|
||||
/// PII detection, lineage with redacted DSN, mark-stale + agent trigger.
|
||||
async fn persist_and_register(
|
||||
parquet: &bytes::Bytes,
|
||||
table: &str,
|
||||
dataset_name_override: Option<&str>,
|
||||
rows: usize,
|
||||
source_system: &str,
|
||||
dsn: &str,
|
||||
deps: &SchedulerIngestDeps,
|
||||
) -> Result<String, String> {
|
||||
use shared::types::{ColumnMeta, Lineage, ObjectRef, SchemaFingerprint, Sensitivity};
|
||||
|
||||
let (schema, _) = shared::arrow_helpers::parquet_to_record_batches(parquet)
|
||||
.map_err(|e| format!("reparse parquet: {e}"))?;
|
||||
|
||||
let dataset_name = dataset_name_override.unwrap_or(table).to_string();
|
||||
let storage_key = format!("datasets/{}.parquet", dataset_name);
|
||||
let parquet_size = parquet.len() as u64;
|
||||
let bucket = "primary".to_string();
|
||||
|
||||
let target_store = deps.buckets.get(&bucket)?;
|
||||
ops::put(&target_store, &storage_key, parquet.clone()).await?;
|
||||
|
||||
let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
|
||||
let now = Utc::now();
|
||||
deps.registry.register(
|
||||
dataset_name.clone(),
|
||||
SchemaFingerprint(schema_fp.0),
|
||||
vec![ObjectRef {
|
||||
bucket: bucket.clone(),
|
||||
key: storage_key.clone(),
|
||||
size_bytes: parquet_size,
|
||||
created_at: now,
|
||||
}],
|
||||
).await?;
|
||||
|
||||
let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
|
||||
let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
|
||||
let columns: Vec<ColumnMeta> = schema.fields().iter().map(|f| {
|
||||
let sens = shared::pii::detect_sensitivity(f.name());
|
||||
ColumnMeta {
|
||||
name: f.name().clone(),
|
||||
data_type: f.data_type().to_string(),
|
||||
sensitivity: sens.clone(),
|
||||
description: String::new(),
|
||||
is_pii: matches!(sens, Some(Sensitivity::Pii)),
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let lineage = Lineage {
|
||||
source_system: source_system.to_string(),
|
||||
source_file: format!("dsn: {}", redact_dsn(dsn)),
|
||||
ingest_job: format!("scheduled-{}-{}", source_system, now.timestamp_millis()),
|
||||
ingest_timestamp: now,
|
||||
parent_datasets: vec![],
|
||||
};
|
||||
|
||||
let _ = deps.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
|
||||
sensitivity,
|
||||
columns: Some(columns),
|
||||
lineage: Some(lineage),
|
||||
row_count: Some(rows as u64),
|
||||
..Default::default()
|
||||
}).await;
|
||||
|
||||
let _ = deps.registry.mark_embeddings_stale(&dataset_name).await;
|
||||
|
||||
if let Some(agent) = &deps.agent_handle {
|
||||
let bound = deps.index_registry.list(Some(&dataset_name), None).await;
|
||||
for meta in bound {
|
||||
let event = vectord::agent::TriggerEvent::dataset_appended(
|
||||
meta.index_name.clone(), &dataset_name,
|
||||
);
|
||||
let _ = agent.enqueue(event).await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(format!("rows={rows} dataset={dataset_name} bytes={parquet_size}"))
|
||||
}
|
||||
|
||||
/// Same generic redactor used in service.rs — duplicated here to avoid
|
||||
/// pulling service::redact_dsn behind a `pub` visibility just for this.
|
||||
fn redact_dsn(dsn: &str) -> String {
|
||||
let scheme_end = match dsn.find("://") {
|
||||
Some(i) => i + 3,
|
||||
None => return dsn.to_string(),
|
||||
};
|
||||
let at_idx = match dsn.rfind('@') {
|
||||
Some(i) if i > scheme_end => i,
|
||||
_ => return dsn.to_string(),
|
||||
};
|
||||
let userpass = &dsn[scheme_end..at_idx];
|
||||
let colon_offset = match userpass.find(':') {
|
||||
Some(i) => i,
|
||||
None => return dsn.to_string(),
|
||||
};
|
||||
let colon_idx = scheme_end + colon_offset;
|
||||
format!("{}:***{}", &dsn[..colon_idx], &dsn[at_idx..])
|
||||
}
|
||||
|
||||
// =================== Tests ===================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Timelike;
|
||||
|
||||
fn mk(secs: u64) -> ScheduleDef {
|
||||
ScheduleDef {
|
||||
id: "test".into(),
|
||||
kind: ScheduleKind::Mysql {
|
||||
dsn: "mysql://x@host/db".into(),
|
||||
table: "t".into(),
|
||||
dataset_name: None,
|
||||
batch_size: None,
|
||||
order_by: None,
|
||||
},
|
||||
trigger: ScheduleTrigger::Interval { secs },
|
||||
enabled: true,
|
||||
created_at: Utc::now(),
|
||||
created_by: "tester".into(),
|
||||
next_run_at: Utc::now(),
|
||||
last_run_at: None,
|
||||
last_outcome: None,
|
||||
run_count: 0,
|
||||
failure_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_due_respects_enabled() {
|
||||
let mut s = mk(60);
|
||||
s.next_run_at = Utc::now() - Duration::seconds(5);
|
||||
assert!(s.is_due(Utc::now()));
|
||||
s.enabled = false;
|
||||
assert!(!s.is_due(Utc::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_due_respects_future_time() {
|
||||
let mut s = mk(60);
|
||||
s.next_run_at = Utc::now() + Duration::seconds(60);
|
||||
assert!(!s.is_due(Utc::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_run_since_last_completion_advances() {
|
||||
let s = mk(120);
|
||||
let completed = Utc::now();
|
||||
let next = compute_next_run_at(&s.trigger, completed, completed);
|
||||
let delta = next - completed;
|
||||
assert_eq!(delta.num_seconds(), 120);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_every_minute_fires_within_sixty_seconds() {
|
||||
let trig = ScheduleTrigger::Cron { expr: "* * * * *".into() };
|
||||
let now = Utc::now();
|
||||
let next = compute_next_run_at(&trig, now, now);
|
||||
let delta = next - now;
|
||||
assert!(delta.num_seconds() > 0 && delta.num_seconds() <= 60,
|
||||
"expected next fire within 60s, got {}s", delta.num_seconds());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_daily_at_1415_utc_is_within_24h() {
|
||||
// 14:15 UTC daily — whether we're before or after 14:15 today, the
|
||||
// next fire is at most 24h out.
|
||||
let trig = ScheduleTrigger::Cron { expr: "15 14 * * *".into() };
|
||||
let now: DateTime<Utc> = "2026-04-20T10:00:00Z".parse().unwrap();
|
||||
let next = compute_next_run_at(&trig, now, now);
|
||||
assert!(next > now);
|
||||
assert!((next - now).num_hours() <= 24);
|
||||
// And the minute-of-hour is 15.
|
||||
assert_eq!(next.minute(), 15);
|
||||
assert_eq!(next.hour(), 14);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_weekday_skips_weekend() {
|
||||
// 09:00 UTC weekdays only. On a Saturday, next fire is Monday.
|
||||
let trig = ScheduleTrigger::Cron { expr: "0 9 * * 1-5".into() };
|
||||
// 2026-04-18 is a Saturday.
|
||||
let sat: DateTime<Utc> = "2026-04-18T10:00:00Z".parse().unwrap();
|
||||
let next = compute_next_run_at(&trig, sat, sat);
|
||||
// Monday 2026-04-20 at 09:00 UTC.
|
||||
assert_eq!(next, "2026-04-20T09:00:00Z".parse::<DateTime<Utc>>().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_six_field_seconds_granularity() {
|
||||
// 6-field (seconds included): fire every minute at 30s past the minute.
|
||||
let trig = ScheduleTrigger::Cron { expr: "30 * * * * *".into() };
|
||||
let now = Utc::now();
|
||||
let next = compute_next_run_at(&trig, now, now);
|
||||
assert_eq!(next.second(), 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_bad_cron() {
|
||||
let bad = ScheduleTrigger::Cron { expr: "not a cron".into() };
|
||||
let err = validate_trigger(&bad).unwrap_err();
|
||||
assert!(err.contains("cron"), "error should mention cron, got: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_wrong_field_count() {
|
||||
// 4 fields — neither 5 nor 6.
|
||||
let bad = ScheduleTrigger::Cron { expr: "* * * *".into() };
|
||||
assert!(validate_trigger(&bad).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_zero_interval() {
|
||||
let bad = ScheduleTrigger::Interval { secs: 0 };
|
||||
assert!(validate_trigger(&bad).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_good_cron() {
|
||||
let good = ScheduleTrigger::Cron { expr: "0 */6 * * *".into() };
|
||||
assert!(validate_trigger(&good).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schedule_kind_label() {
|
||||
let k = ScheduleKind::Mysql {
|
||||
dsn: "x".into(), table: "customers".into(),
|
||||
dataset_name: None, batch_size: None, order_by: None,
|
||||
};
|
||||
assert_eq!(k.label(), "mysql:customers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_dsn_handles_mysql_and_postgres() {
|
||||
assert_eq!(redact_dsn("mysql://u:secret@h/db"), "mysql://u:***@h/db");
|
||||
assert_eq!(redact_dsn("postgresql://u:secret@h:5432/db"), "postgresql://u:***@h:5432/db");
|
||||
assert_eq!(redact_dsn("mysql://u@h/db"), "mysql://u@h/db"); // no password
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/// Schema evolution: detect changes between ingests and suggest migration rules.
|
||||
/// When a source changes format (columns renamed, added, removed, type changed),
|
||||
/// the system detects the diff and can auto-map using AI or heuristic matching.
|
||||
|
||||
use arrow::datatypes::{DataType, SchemaRef};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// A detected change between two schema versions.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SchemaChange {
|
||||
pub change_type: ChangeType,
|
||||
pub column: String,
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ChangeType {
|
||||
Added, // new column in incoming schema
|
||||
Removed, // column in old schema not in new
|
||||
TypeChanged, // same name, different type
|
||||
Renamed, // likely renamed (fuzzy match)
|
||||
}
|
||||
|
||||
/// A migration rule — how to map old schema to new.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MigrationRule {
|
||||
pub source_column: String,
|
||||
pub target_column: String,
|
||||
pub transform: String, // "direct", "cast", "concat", "split", "drop"
|
||||
pub confidence: f32, // 0.0 - 1.0
|
||||
pub auto_generated: bool,
|
||||
}
|
||||
|
||||
/// Compare two schemas and detect changes.
|
||||
pub fn diff_schemas(old: &SchemaRef, new: &SchemaRef) -> Vec<SchemaChange> {
|
||||
let mut changes = Vec::new();
|
||||
|
||||
let old_fields: HashMap<&str, &DataType> = old.fields().iter()
|
||||
.map(|f| (f.name().as_str(), f.data_type()))
|
||||
.collect();
|
||||
|
||||
let new_fields: HashMap<&str, &DataType> = new.fields().iter()
|
||||
.map(|f| (f.name().as_str(), f.data_type()))
|
||||
.collect();
|
||||
|
||||
// Check for removed columns
|
||||
for (name, _dt) in &old_fields {
|
||||
if !new_fields.contains_key(name) {
|
||||
// Check if it was renamed (fuzzy match)
|
||||
let possible_rename = find_similar_column(name, &new_fields, &old_fields);
|
||||
if let Some((new_name, confidence)) = possible_rename {
|
||||
changes.push(SchemaChange {
|
||||
change_type: ChangeType::Renamed,
|
||||
column: name.to_string(),
|
||||
detail: format!("likely renamed to '{}' (confidence: {:.0}%)", new_name, confidence * 100.0),
|
||||
});
|
||||
} else {
|
||||
changes.push(SchemaChange {
|
||||
change_type: ChangeType::Removed,
|
||||
column: name.to_string(),
|
||||
detail: "column no longer present in incoming data".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for added or type-changed columns
|
||||
for (name, new_dt) in &new_fields {
|
||||
match old_fields.get(name) {
|
||||
None => {
|
||||
// Skip if we already flagged this as a rename target
|
||||
let is_rename_target = changes.iter().any(|c| {
|
||||
c.change_type == ChangeType::Renamed && c.detail.contains(name)
|
||||
});
|
||||
if !is_rename_target {
|
||||
changes.push(SchemaChange {
|
||||
change_type: ChangeType::Added,
|
||||
column: name.to_string(),
|
||||
detail: format!("new column (type: {})", new_dt),
|
||||
});
|
||||
}
|
||||
}
|
||||
Some(old_dt) => {
|
||||
if old_dt != new_dt {
|
||||
changes.push(SchemaChange {
|
||||
change_type: ChangeType::TypeChanged,
|
||||
column: name.to_string(),
|
||||
detail: format!("{} → {}", old_dt, new_dt),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Generate migration rules from detected changes.
|
||||
pub fn generate_rules(changes: &[SchemaChange]) -> Vec<MigrationRule> {
|
||||
changes.iter().filter_map(|change| {
|
||||
match change.change_type {
|
||||
ChangeType::Renamed => {
|
||||
// Extract the new name from the detail string
|
||||
let new_name = change.detail
|
||||
.split('\'')
|
||||
.nth(1)
|
||||
.unwrap_or(&change.column);
|
||||
Some(MigrationRule {
|
||||
source_column: change.column.clone(),
|
||||
target_column: new_name.to_string(),
|
||||
transform: "direct".into(),
|
||||
confidence: 0.8,
|
||||
auto_generated: true,
|
||||
})
|
||||
}
|
||||
ChangeType::TypeChanged => {
|
||||
Some(MigrationRule {
|
||||
source_column: change.column.clone(),
|
||||
target_column: change.column.clone(),
|
||||
transform: "cast".into(),
|
||||
confidence: 0.9,
|
||||
auto_generated: true,
|
||||
})
|
||||
}
|
||||
ChangeType::Added => {
|
||||
Some(MigrationRule {
|
||||
source_column: String::new(),
|
||||
target_column: change.column.clone(),
|
||||
transform: "new_column".into(),
|
||||
confidence: 1.0,
|
||||
auto_generated: true,
|
||||
})
|
||||
}
|
||||
ChangeType::Removed => {
|
||||
Some(MigrationRule {
|
||||
source_column: change.column.clone(),
|
||||
target_column: String::new(),
|
||||
transform: "drop".into(),
|
||||
confidence: 0.7,
|
||||
auto_generated: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
}).collect()
|
||||
}
|
||||
|
||||
/// Build an AI prompt to suggest migration rules for complex cases.
|
||||
pub fn build_migration_prompt(
|
||||
old_schema: &SchemaRef,
|
||||
new_schema: &SchemaRef,
|
||||
changes: &[SchemaChange],
|
||||
) -> String {
|
||||
let old_cols: Vec<String> = old_schema.fields().iter()
|
||||
.map(|f| format!(" {} ({})", f.name(), f.data_type()))
|
||||
.collect();
|
||||
let new_cols: Vec<String> = new_schema.fields().iter()
|
||||
.map(|f| format!(" {} ({})", f.name(), f.data_type()))
|
||||
.collect();
|
||||
let change_list: Vec<String> = changes.iter()
|
||||
.map(|c| format!(" [{:?}] {} — {}", c.change_type, c.column, c.detail))
|
||||
.collect();
|
||||
|
||||
format!(
|
||||
"You are a data migration expert. A data source has changed its schema.\n\n\
|
||||
OLD SCHEMA:\n{}\n\n\
|
||||
NEW SCHEMA:\n{}\n\n\
|
||||
DETECTED CHANGES:\n{}\n\n\
|
||||
For each change, suggest a migration rule as JSON:\n\
|
||||
{{\"source_column\": \"old_name\", \"target_column\": \"new_name\", \"transform\": \"direct|cast|concat|split|drop\", \"confidence\": 0.0-1.0}}\n\n\
|
||||
Output ONLY a JSON array of rules, nothing else.",
|
||||
old_cols.join("\n"),
|
||||
new_cols.join("\n"),
|
||||
change_list.join("\n"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Fuzzy match: find a column in new schema that's similar to a removed old column.
|
||||
fn find_similar_column<'a>(
|
||||
old_name: &str,
|
||||
new_fields: &HashMap<&'a str, &DataType>,
|
||||
old_fields: &HashMap<&str, &DataType>,
|
||||
) -> Option<(String, f32)> {
|
||||
let old_lower = old_name.to_lowercase();
|
||||
let old_parts: Vec<&str> = old_lower.split('_').collect();
|
||||
|
||||
let mut best_match: Option<(String, f32)> = None;
|
||||
|
||||
for (new_name, _) in new_fields {
|
||||
// Skip columns that already exist in old schema (not renamed)
|
||||
if old_fields.contains_key(new_name) { continue; }
|
||||
|
||||
let new_lower = new_name.to_lowercase();
|
||||
let new_parts: Vec<&str> = new_lower.split('_').collect();
|
||||
|
||||
// Exact substring match
|
||||
if old_lower.contains(&new_lower) || new_lower.contains(&old_lower) {
|
||||
let confidence = 0.85;
|
||||
if best_match.as_ref().map_or(true, |(_, c)| confidence > *c) {
|
||||
best_match = Some((new_name.to_string(), confidence));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Shared word parts (e.g., "first_name" → "fname", "full_name")
|
||||
let shared: usize = old_parts.iter()
|
||||
.filter(|p| new_parts.iter().any(|np| np.contains(*p) || p.contains(np)))
|
||||
.count();
|
||||
|
||||
if shared > 0 {
|
||||
let confidence = shared as f32 / old_parts.len().max(new_parts.len()) as f32 * 0.75;
|
||||
if confidence >= 0.3 && best_match.as_ref().map_or(true, |(_, c)| confidence > *c) {
|
||||
best_match = Some((new_name.to_string(), confidence));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
best_match
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use arrow::datatypes::{Field, Schema};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn schema(fields: Vec<(&str, DataType)>) -> SchemaRef {
|
||||
Arc::new(Schema::new(
|
||||
fields.into_iter().map(|(n, t)| Field::new(n, t, true)).collect::<Vec<_>>()
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_added_column() {
|
||||
let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
|
||||
let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("email", DataType::Utf8)]);
|
||||
let changes = diff_schemas(&old, &new);
|
||||
assert_eq!(changes.len(), 1);
|
||||
assert_eq!(changes[0].change_type, ChangeType::Added);
|
||||
assert_eq!(changes[0].column, "email");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_removed_column() {
|
||||
let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("age", DataType::Int64)]);
|
||||
let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
|
||||
let changes = diff_schemas(&old, &new);
|
||||
assert!(changes.iter().any(|c| c.column == "age" && c.change_type == ChangeType::Removed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_type_change() {
|
||||
let old = schema(vec![("id", DataType::Int64), ("score", DataType::Int64)]);
|
||||
let new = schema(vec![("id", DataType::Int64), ("score", DataType::Float64)]);
|
||||
let changes = diff_schemas(&old, &new);
|
||||
assert_eq!(changes.len(), 1);
|
||||
assert_eq!(changes[0].change_type, ChangeType::TypeChanged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_rename() {
|
||||
let old = schema(vec![("first_name", DataType::Utf8), ("last_name", DataType::Utf8)]);
|
||||
let new = schema(vec![("full_name", DataType::Utf8)]);
|
||||
let changes = diff_schemas(&old, &new);
|
||||
// Should detect first_name → full_name as a potential rename
|
||||
assert!(changes.iter().any(|c| c.change_type == ChangeType::Renamed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_rules_from_changes() {
|
||||
let old = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8)]);
|
||||
let new = schema(vec![("id", DataType::Int64), ("name", DataType::Utf8), ("email", DataType::Utf8)]);
|
||||
let changes = diff_schemas(&old, &new);
|
||||
let rules = generate_rules(&changes);
|
||||
assert_eq!(rules.len(), 1);
|
||||
assert_eq!(rules[0].transform, "new_column");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,758 @@
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Multipart, Path, Query, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use object_store::ObjectStore;
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
use catalogd::registry::Registry;
|
||||
use crate::{db_ingest, my_stream, pg_stream, pipeline, schedule};
|
||||
use shared::arrow_helpers::record_batch_to_parquet;
|
||||
use shared::types::{ObjectRef, SchemaFingerprint};
|
||||
use storaged::ops;
|
||||
use storaged::registry::BucketRegistry;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct IngestState {
|
||||
pub store: Arc<dyn ObjectStore>,
|
||||
pub registry: Registry,
|
||||
/// Federation layer 2: lookup target bucket from request headers.
|
||||
pub buckets: Arc<BucketRegistry>,
|
||||
/// Phase 16.5: when ingest marks a dataset's embeddings stale, push
|
||||
/// a `DatasetAppended` trigger so the autotune agent can schedule
|
||||
/// re-trials. Holds the agent handle (no-op when agent disabled).
|
||||
pub agent_handle: vectord::agent::AgentHandle,
|
||||
/// Used to look up which HNSW indexes are attached to the
|
||||
/// just-ingested dataset. Each matching index gets one trigger.
|
||||
pub index_registry: vectord::index_registry::IndexRegistry,
|
||||
/// Scheduled-ingest registry. The scheduler task runs against this
|
||||
/// store; HTTP CRUD endpoints write through it.
|
||||
pub schedules: schedule::ScheduleStore,
|
||||
/// Event journal for ADR-012 mutation history. Optional for back-compat
|
||||
/// with callers (like scheduled ingest tests) that don't wire it yet.
|
||||
/// When present, successful ingests emit a record_ingest event — closes
|
||||
/// P9-001 on the file-upload path. (2026-04-23)
|
||||
pub journal: Option<journald::journal::Journal>,
|
||||
}
|
||||
|
||||
/// Push `DatasetAppended` triggers for every HNSW index bound to this
|
||||
/// dataset. Called after a successful ingest. Logs failures but never
|
||||
/// fails the ingest — the agent is best-effort, not load-bearing.
|
||||
async fn notify_agent_on_append(state: &IngestState, dataset_name: &str) {
|
||||
let indexes = state.index_registry.list(Some(dataset_name), None).await;
|
||||
if indexes.is_empty() {
|
||||
return;
|
||||
}
|
||||
for meta in indexes {
|
||||
let event = vectord::agent::TriggerEvent::dataset_appended(
|
||||
meta.index_name.clone(), dataset_name,
|
||||
);
|
||||
if let Err(e) = state.agent_handle.enqueue(event).await {
|
||||
tracing::warn!("agent enqueue failed for '{}': {}", meta.index_name, e);
|
||||
} else {
|
||||
tracing::info!(
|
||||
"agent: enqueued DatasetAppended({}) for index '{}'",
|
||||
dataset_name, meta.index_name,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the target bucket from `X-Lakehouse-Bucket` header.
|
||||
/// Returns `(bucket_name, store_for_writes)`. Falls back to "primary"
|
||||
/// when the header is absent. Returns Err with the canonical bucket
|
||||
/// list when the header names an unknown bucket.
|
||||
fn resolve_bucket(
|
||||
headers: &HeaderMap,
|
||||
buckets: &BucketRegistry,
|
||||
) -> Result<(String, Arc<dyn ObjectStore>), (StatusCode, String)> {
|
||||
let target = headers
|
||||
.get("x-lakehouse-bucket")
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| buckets.default_name().to_string());
|
||||
|
||||
match buckets.get(&target) {
|
||||
Ok(store) => Ok((target, store)),
|
||||
Err(_) => Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
format!("unknown bucket '{}' — use GET /storage/buckets to list", target),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn router(state: IngestState) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/file", post(ingest_file))
|
||||
.route("/postgres/tables", post(list_pg_tables))
|
||||
.route("/postgres/import", post(import_pg_table))
|
||||
.route("/db", post(ingest_db_stream))
|
||||
.route("/mysql", post(ingest_mysql_stream))
|
||||
// Phase E: scheduled ingest
|
||||
.route("/schedules", get(list_schedules).post(create_schedule))
|
||||
.route("/schedules/{id}", get(get_schedule)
|
||||
.patch(patch_schedule)
|
||||
.delete(delete_schedule))
|
||||
.route("/schedules/{id}/run-now", post(run_schedule_now))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
async fn health() -> &'static str {
|
||||
"ingestd ok"
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct IngestQuery {
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
async fn ingest_file(
|
||||
State(state): State<IngestState>,
|
||||
Query(query): Query<IngestQuery>,
|
||||
headers: HeaderMap,
|
||||
mut multipart: Multipart,
|
||||
) -> impl IntoResponse {
|
||||
let (bucket, store) = resolve_bucket(&headers, &state.buckets)?;
|
||||
|
||||
let field = match multipart.next_field().await {
|
||||
Ok(Some(f)) => f,
|
||||
Ok(None) => return Err((StatusCode::BAD_REQUEST, "no file uploaded".to_string())),
|
||||
Err(e) => return Err((StatusCode::BAD_REQUEST, format!("multipart error: {e}"))),
|
||||
};
|
||||
|
||||
let filename = field.file_name().unwrap_or("unknown").to_string();
|
||||
let content = field.bytes().await
|
||||
.map_err(|e| (StatusCode::BAD_REQUEST, format!("read error: {e}")))?;
|
||||
|
||||
tracing::info!(
|
||||
"ingest '{}' ({} bytes) -> bucket={}",
|
||||
filename, content.len(), bucket,
|
||||
);
|
||||
|
||||
let dataset_name = query.name.as_deref();
|
||||
|
||||
match pipeline::ingest_file_to_bucket(&filename, &content, dataset_name, &bucket, &store, &state.registry).await {
|
||||
Ok(result) => {
|
||||
if !result.deduplicated {
|
||||
notify_agent_on_append(&state, &result.dataset_name).await;
|
||||
// P9-001 fix (2026-04-23): emit a mutation event on every
|
||||
// non-deduplicated ingest. Dedup no-ops don't need events
|
||||
// (ADR-020 register() is already idempotent on same fingerprint).
|
||||
if let Some(ref journal) = state.journal {
|
||||
if let Err(e) = journal.record_ingest(
|
||||
&result.dataset_name,
|
||||
result.rows as usize,
|
||||
"ingest_api",
|
||||
&filename,
|
||||
).await {
|
||||
tracing::warn!(
|
||||
"journal record_ingest failed for '{}': {}",
|
||||
result.dataset_name, e,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.deduplicated {
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
} else {
|
||||
Ok((StatusCode::CREATED, Json(result)))
|
||||
}
|
||||
}
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
// --- PostgreSQL Import ---
|
||||
|
||||
/// List tables in a PostgreSQL database.
|
||||
async fn list_pg_tables(
|
||||
Json(config): Json<db_ingest::DbConfig>,
|
||||
) -> impl IntoResponse {
|
||||
match db_ingest::list_postgres_tables(&config).await {
|
||||
Ok(tables) => Ok(Json(tables)),
|
||||
Err(e) => Err((StatusCode::BAD_GATEWAY, e)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PgImportRequest {
|
||||
#[serde(flatten)]
|
||||
config: db_ingest::DbConfig,
|
||||
table: String,
|
||||
/// Override dataset name (defaults to table name)
|
||||
dataset_name: Option<String>,
|
||||
/// Max rows to import (None = all)
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
/// Import a PostgreSQL table into the lakehouse.
|
||||
async fn import_pg_table(
|
||||
State(state): State<IngestState>,
|
||||
Json(req): Json<PgImportRequest>,
|
||||
) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
|
||||
tracing::info!("importing postgres table '{}' from {}:{}/{}",
|
||||
req.table, req.config.host, req.config.port, req.config.database);
|
||||
|
||||
// Import from Postgres
|
||||
let (schema, batches, db_result) = db_ingest::import_postgres_table(
|
||||
&req.config, &req.table, req.limit,
|
||||
).await.map_err(|e| (StatusCode::BAD_GATEWAY, e))?;
|
||||
|
||||
if batches.is_empty() || db_result.rows == 0 {
|
||||
return Ok((StatusCode::OK, Json(serde_json::json!({
|
||||
"table": req.table,
|
||||
"rows": 0,
|
||||
"message": "table is empty",
|
||||
}))));
|
||||
}
|
||||
|
||||
// Convert to Parquet
|
||||
let mut all_parquet = Vec::new();
|
||||
for batch in &batches {
|
||||
let pq = record_batch_to_parquet(batch)
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
all_parquet.extend_from_slice(&pq);
|
||||
}
|
||||
|
||||
let dataset_name = req.dataset_name.unwrap_or_else(|| req.table.clone());
|
||||
let storage_key = format!("datasets/{}.parquet", dataset_name);
|
||||
let parquet_size = all_parquet.len() as u64;
|
||||
|
||||
// Store
|
||||
ops::put(&state.store, &storage_key, Bytes::from(all_parquet))
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
|
||||
// Register
|
||||
let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
|
||||
let now = chrono::Utc::now();
|
||||
state.registry.register(
|
||||
dataset_name.clone(),
|
||||
SchemaFingerprint(schema_fp.0),
|
||||
vec![ObjectRef {
|
||||
bucket: "data".to_string(),
|
||||
key: storage_key.clone(),
|
||||
size_bytes: parquet_size,
|
||||
created_at: now,
|
||||
}],
|
||||
).await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
|
||||
// Auto-populate metadata (PII detection, lineage)
|
||||
let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
|
||||
let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
|
||||
let columns: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
|
||||
let sens = shared::pii::detect_sensitivity(f.name());
|
||||
shared::types::ColumnMeta {
|
||||
name: f.name().clone(),
|
||||
data_type: f.data_type().to_string(),
|
||||
sensitivity: sens.clone(),
|
||||
description: String::new(),
|
||||
is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let lineage = shared::types::Lineage {
|
||||
source_system: "postgresql".to_string(),
|
||||
source_file: format!("{}:{}/{}.{}", req.config.host, req.config.port, req.config.database, req.table),
|
||||
ingest_job: format!("pg-import-{}", now.timestamp_millis()),
|
||||
ingest_timestamp: now,
|
||||
parent_datasets: vec![],
|
||||
};
|
||||
|
||||
let _ = state.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
|
||||
sensitivity,
|
||||
columns: Some(columns),
|
||||
lineage: Some(lineage),
|
||||
row_count: Some(db_result.rows as u64),
|
||||
..Default::default()
|
||||
}).await;
|
||||
|
||||
tracing::info!("imported '{}' from postgres: {} rows → {}", dataset_name, db_result.rows, storage_key);
|
||||
|
||||
Ok((StatusCode::CREATED, Json(serde_json::json!({
|
||||
"dataset_name": dataset_name,
|
||||
"table": req.table,
|
||||
"rows": db_result.rows,
|
||||
"columns": db_result.columns,
|
||||
"schema": db_result.schema_detected,
|
||||
"storage_key": storage_key,
|
||||
"size_bytes": parquet_size,
|
||||
}))))
|
||||
}
|
||||
|
||||
/// Streaming DSN-based ingest. Paginates via ORDER BY + LIMIT/OFFSET so
|
||||
/// large tables don't blow up memory. This is the path the task spec calls
|
||||
/// `POST /ingest/db` with `{dsn, table, dataset_name}`.
|
||||
async fn ingest_db_stream(
|
||||
State(state): State<IngestState>,
|
||||
headers: HeaderMap,
|
||||
Json(req): Json<pg_stream::PgStreamRequest>,
|
||||
) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
|
||||
let (bucket, store) = resolve_bucket(&headers, &state.buckets)?;
|
||||
tracing::info!(
|
||||
"pg stream ingest: table='{}' dataset='{:?}' bucket='{}' batch_size={:?}",
|
||||
req.table, req.dataset_name, bucket, req.batch_size,
|
||||
);
|
||||
|
||||
// Stream from Postgres into Parquet bytes.
|
||||
let (parquet, stream_result) = pg_stream::stream_table_to_parquet(&req)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, e))?;
|
||||
|
||||
if stream_result.rows == 0 {
|
||||
return Ok((StatusCode::OK, Json(serde_json::json!({
|
||||
"table": req.table,
|
||||
"rows": 0,
|
||||
"message": "table is empty",
|
||||
}))));
|
||||
}
|
||||
|
||||
// Recover schema from the parquet footer — keeps a single source of truth
|
||||
// for the schema fingerprint + column metadata.
|
||||
let (schema, _) = shared::arrow_helpers::parquet_to_record_batches(&parquet)
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("reparse parquet: {e}")))?;
|
||||
|
||||
let dataset_name = req.dataset_name.clone().unwrap_or_else(|| req.table.clone());
|
||||
let storage_key = format!("datasets/{}.parquet", dataset_name);
|
||||
let size_bytes = parquet.len() as u64;
|
||||
|
||||
ops::put(&store, &storage_key, parquet)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
|
||||
let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
|
||||
let now = chrono::Utc::now();
|
||||
state.registry.register(
|
||||
dataset_name.clone(),
|
||||
SchemaFingerprint(schema_fp.0),
|
||||
vec![ObjectRef {
|
||||
bucket: bucket.clone(),
|
||||
key: storage_key.clone(),
|
||||
size_bytes,
|
||||
created_at: now,
|
||||
}],
|
||||
).await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
|
||||
// Rich metadata: PII auto-detect, lineage, row_count, columns.
|
||||
let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
|
||||
let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
|
||||
let columns: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
|
||||
let sens = shared::pii::detect_sensitivity(f.name());
|
||||
shared::types::ColumnMeta {
|
||||
name: f.name().clone(),
|
||||
data_type: f.data_type().to_string(),
|
||||
sensitivity: sens.clone(),
|
||||
description: String::new(),
|
||||
is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let lineage = shared::types::Lineage {
|
||||
source_system: "postgresql".to_string(),
|
||||
source_file: format!("dsn: {}", redact_dsn(&req.dsn)),
|
||||
ingest_job: format!("pg-stream-{}", now.timestamp_millis()),
|
||||
ingest_timestamp: now,
|
||||
parent_datasets: vec![],
|
||||
};
|
||||
|
||||
let _ = state.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
|
||||
sensitivity,
|
||||
columns: Some(columns),
|
||||
lineage: Some(lineage),
|
||||
row_count: Some(stream_result.rows as u64),
|
||||
..Default::default()
|
||||
}).await;
|
||||
|
||||
// Phase C: mark embeddings stale if the dataset already had a vector
|
||||
// index. No-op for newly-created datasets.
|
||||
let _ = state.registry.mark_embeddings_stale(&dataset_name).await;
|
||||
|
||||
// Phase 16.5: notify autotune agent. No-op if no indexes attached.
|
||||
notify_agent_on_append(&state, &dataset_name).await;
|
||||
|
||||
Ok((StatusCode::CREATED, Json(serde_json::json!({
|
||||
"dataset_name": dataset_name,
|
||||
"table": stream_result.table,
|
||||
"rows": stream_result.rows,
|
||||
"batches": stream_result.batches,
|
||||
"columns": stream_result.columns,
|
||||
"schema": stream_result.schema,
|
||||
"storage_key": storage_key,
|
||||
"size_bytes": size_bytes,
|
||||
"duration_secs": stream_result.duration_secs,
|
||||
}))))
|
||||
}
|
||||
|
||||
/// Streaming MySQL ingest. Mirrors `ingest_db_stream` — same pagination +
|
||||
/// Parquet-footer schema recovery + PII/lineage metadata. Different driver,
|
||||
/// otherwise identical.
|
||||
async fn ingest_mysql_stream(
|
||||
State(state): State<IngestState>,
|
||||
headers: HeaderMap,
|
||||
Json(req): Json<my_stream::MyStreamRequest>,
|
||||
) -> Result<(StatusCode, Json<serde_json::Value>), (StatusCode, String)> {
|
||||
let (bucket, store) = resolve_bucket(&headers, &state.buckets)?;
|
||||
tracing::info!(
|
||||
"mysql stream ingest: table='{}' dataset='{:?}' bucket='{}' batch_size={:?}",
|
||||
req.table, req.dataset_name, bucket, req.batch_size,
|
||||
);
|
||||
|
||||
let (parquet, stream_result) = my_stream::stream_table_to_parquet(&req)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, e))?;
|
||||
|
||||
if stream_result.rows == 0 {
|
||||
return Ok((StatusCode::OK, Json(serde_json::json!({
|
||||
"table": req.table,
|
||||
"rows": 0,
|
||||
"message": "table is empty",
|
||||
}))));
|
||||
}
|
||||
|
||||
let (schema, _) = shared::arrow_helpers::parquet_to_record_batches(&parquet)
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("reparse parquet: {e}")))?;
|
||||
|
||||
let dataset_name = req.dataset_name.clone().unwrap_or_else(|| req.table.clone());
|
||||
let storage_key = format!("datasets/{}.parquet", dataset_name);
|
||||
let size_bytes = parquet.len() as u64;
|
||||
|
||||
ops::put(&store, &storage_key, parquet)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
|
||||
let schema_fp = shared::arrow_helpers::fingerprint_schema(&schema);
|
||||
let now = chrono::Utc::now();
|
||||
state.registry.register(
|
||||
dataset_name.clone(),
|
||||
SchemaFingerprint(schema_fp.0),
|
||||
vec![ObjectRef {
|
||||
bucket: bucket.clone(),
|
||||
key: storage_key.clone(),
|
||||
size_bytes,
|
||||
created_at: now,
|
||||
}],
|
||||
).await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?;
|
||||
|
||||
let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
|
||||
let sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
|
||||
let columns: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
|
||||
let sens = shared::pii::detect_sensitivity(f.name());
|
||||
shared::types::ColumnMeta {
|
||||
name: f.name().clone(),
|
||||
data_type: f.data_type().to_string(),
|
||||
sensitivity: sens.clone(),
|
||||
description: String::new(),
|
||||
is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let lineage = shared::types::Lineage {
|
||||
source_system: "mysql".to_string(),
|
||||
source_file: format!("dsn: {}", redact_dsn(&req.dsn)),
|
||||
ingest_job: format!("mysql-stream-{}", now.timestamp_millis()),
|
||||
ingest_timestamp: now,
|
||||
parent_datasets: vec![],
|
||||
};
|
||||
|
||||
let _ = state.registry.update_metadata(&dataset_name, catalogd::registry::MetadataUpdate {
|
||||
sensitivity,
|
||||
columns: Some(columns),
|
||||
lineage: Some(lineage),
|
||||
row_count: Some(stream_result.rows as u64),
|
||||
..Default::default()
|
||||
}).await;
|
||||
|
||||
// Phase C: mark stale + Phase 16.5: notify agent.
|
||||
let _ = state.registry.mark_embeddings_stale(&dataset_name).await;
|
||||
notify_agent_on_append(&state, &dataset_name).await;
|
||||
|
||||
Ok((StatusCode::CREATED, Json(serde_json::json!({
|
||||
"dataset_name": dataset_name,
|
||||
"table": stream_result.table,
|
||||
"rows": stream_result.rows,
|
||||
"batches": stream_result.batches,
|
||||
"columns": stream_result.columns,
|
||||
"schema": stream_result.schema,
|
||||
"storage_key": storage_key,
|
||||
"size_bytes": size_bytes,
|
||||
"duration_secs": stream_result.duration_secs,
|
||||
}))))
|
||||
}
|
||||
|
||||
/// Redact the password in any `scheme://user:pass@host/...` DSN. Works
|
||||
/// for postgres, mysql, and any other scheme using the same shape.
|
||||
fn redact_dsn(dsn: &str) -> String {
|
||||
let scheme_end = match dsn.find("://") {
|
||||
Some(i) => i + 3,
|
||||
None => return dsn.to_string(),
|
||||
};
|
||||
let at_idx = match dsn.rfind('@') {
|
||||
Some(i) if i > scheme_end => i,
|
||||
_ => return dsn.to_string(),
|
||||
};
|
||||
// Find the password separator (first `:` after scheme, before `@`).
|
||||
let userpass = &dsn[scheme_end..at_idx];
|
||||
let colon_offset = match userpass.find(':') {
|
||||
Some(i) => i,
|
||||
None => return dsn.to_string(), // no password in DSN
|
||||
};
|
||||
let colon_idx = scheme_end + colon_offset;
|
||||
format!("{}:***{}", &dsn[..colon_idx], &dsn[at_idx..])
|
||||
}
|
||||
|
||||
// --- Phase E: Scheduled ingest CRUD endpoints ---
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateScheduleRequest {
|
||||
/// Stable id used in URLs and on disk. Required so callers can
|
||||
/// PATCH/DELETE deterministically. Sanitized on storage but the
|
||||
/// caller's id is what gets persisted as the canonical key.
|
||||
id: String,
|
||||
kind: schedule::ScheduleKind,
|
||||
trigger: schedule::ScheduleTrigger,
|
||||
#[serde(default = "default_enabled")]
|
||||
enabled: bool,
|
||||
#[serde(default)]
|
||||
created_by: String,
|
||||
}
|
||||
|
||||
fn default_enabled() -> bool { true }
|
||||
|
||||
async fn create_schedule(
|
||||
State(state): State<IngestState>,
|
||||
Json(req): Json<CreateScheduleRequest>,
|
||||
) -> impl IntoResponse {
|
||||
if state.schedules.get(&req.id).await.is_some() {
|
||||
return Err((StatusCode::CONFLICT, format!("schedule '{}' already exists", req.id)));
|
||||
}
|
||||
if let Err(e) = schedule::validate_trigger(&req.trigger) {
|
||||
return Err((StatusCode::BAD_REQUEST, e));
|
||||
}
|
||||
let now = chrono::Utc::now();
|
||||
let def = schedule::ScheduleDef {
|
||||
id: req.id,
|
||||
kind: req.kind,
|
||||
trigger: req.trigger,
|
||||
enabled: req.enabled,
|
||||
created_at: now,
|
||||
created_by: req.created_by,
|
||||
// Fire on the next tick — operators usually want to verify
|
||||
// their schedule works without waiting for the full interval.
|
||||
next_run_at: now,
|
||||
last_run_at: None,
|
||||
last_outcome: None,
|
||||
run_count: 0,
|
||||
failure_count: 0,
|
||||
};
|
||||
match state.schedules.put(def).await {
|
||||
Ok(d) => Ok((StatusCode::CREATED, Json(d))),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_schedules(State(state): State<IngestState>) -> impl IntoResponse {
|
||||
Json(state.schedules.list().await)
|
||||
}
|
||||
|
||||
async fn get_schedule(
|
||||
State(state): State<IngestState>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match state.schedules.get(&id).await {
|
||||
Some(d) => Ok(Json(d)),
|
||||
None => Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found"))),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PatchScheduleRequest {
|
||||
/// Toggle on/off without changing anything else.
|
||||
#[serde(default)]
|
||||
enabled: Option<bool>,
|
||||
/// Replace the trigger spec — useful for bumping intervals.
|
||||
#[serde(default)]
|
||||
trigger: Option<schedule::ScheduleTrigger>,
|
||||
}
|
||||
|
||||
async fn patch_schedule(
|
||||
State(state): State<IngestState>,
|
||||
Path(id): Path<String>,
|
||||
Json(req): Json<PatchScheduleRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let Some(mut def) = state.schedules.get(&id).await else {
|
||||
return Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found")));
|
||||
};
|
||||
if let Some(t) = &req.trigger {
|
||||
if let Err(e) = schedule::validate_trigger(t) {
|
||||
return Err((StatusCode::BAD_REQUEST, e));
|
||||
}
|
||||
}
|
||||
if let Some(e) = req.enabled { def.enabled = e; }
|
||||
if let Some(t) = req.trigger { def.trigger = t; }
|
||||
match state.schedules.put(def).await {
|
||||
Ok(d) => Ok(Json(d)),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_schedule(
|
||||
State(state): State<IngestState>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match state.schedules.delete(&id).await {
|
||||
Ok(true) => Ok(Json(serde_json::json!({ "deleted": id }))),
|
||||
Ok(false) => Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found"))),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Force a schedule to fire right now, regardless of its next_run_at.
|
||||
/// Records the outcome and reschedules. Useful for "test my new schedule
|
||||
/// works without waiting an hour."
|
||||
async fn run_schedule_now(
|
||||
State(state): State<IngestState>,
|
||||
Path(id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
let Some(def) = state.schedules.get(&id).await else {
|
||||
return Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found")));
|
||||
};
|
||||
let deps = schedule::SchedulerIngestDeps {
|
||||
store: state.store.clone(),
|
||||
registry: state.registry.clone(),
|
||||
buckets: state.buckets.clone(),
|
||||
agent_handle: Some(state.agent_handle.clone()),
|
||||
index_registry: state.index_registry.clone(),
|
||||
};
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let result = schedule::run_schedule(&def, &deps).await;
|
||||
let elapsed = t0.elapsed().as_secs_f32();
|
||||
let outcome = schedule::RunOutcome {
|
||||
at: chrono::Utc::now(),
|
||||
success: result.is_ok(),
|
||||
message: match &result {
|
||||
Ok(s) => s.clone(),
|
||||
Err(e) => e.clone(),
|
||||
},
|
||||
rows: result.as_ref().ok().and_then(|m: &String| {
|
||||
m.split_whitespace()
|
||||
.find_map(|tok| tok.strip_prefix("rows=").and_then(|n| n.parse().ok()))
|
||||
}),
|
||||
duration_secs: elapsed,
|
||||
};
|
||||
if let Err(e) = state.schedules.record_run(&id, outcome.clone()).await {
|
||||
tracing::warn!("schedule {id}: record_run failed: {e}");
|
||||
}
|
||||
Ok(Json(outcome))
|
||||
}
|
||||
|
||||
// ─── Tests ───
|
||||
|
||||
#[cfg(test)]
|
||||
mod journal_integration_tests {
|
||||
//! P9-001 integration test: prove that a successful ingest produces a
|
||||
//! journal.record_ingest event. Block 2 on PR #10 was "journal event
|
||||
//! verified live" being unbacked by the diff. This test makes the
|
||||
//! verification committed and reproducible.
|
||||
|
||||
use journald::journal::{Event, Journal};
|
||||
use object_store::memory::InMemory;
|
||||
use std::sync::Arc;
|
||||
|
||||
// Helper: build a bare Journal against an in-memory object store.
|
||||
// Flush threshold 1 so every recorded event is persisted immediately.
|
||||
fn test_journal() -> Journal {
|
||||
let store: Arc<dyn object_store::ObjectStore> = Arc::new(InMemory::new());
|
||||
Journal::new(store, 1)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_record_ingest_increments_counter() {
|
||||
// Arrange — fresh journal, counter starts at zero.
|
||||
let journal = test_journal();
|
||||
let stats0 = journal.stats().await;
|
||||
assert_eq!(stats0.total_events_created, 0);
|
||||
assert_eq!(stats0.buffer_events, 0);
|
||||
|
||||
// Act — simulate what the /ingest/file success path does.
|
||||
journal
|
||||
.record_ingest("test_dataset", 42, "ingest_api", "probe.csv")
|
||||
.await
|
||||
.expect("record_ingest should succeed");
|
||||
|
||||
// Assert — counter advanced, event exists. With threshold=1 the
|
||||
// event flushed to store; with threshold>N it would be in-buffer.
|
||||
let stats1 = journal.stats().await;
|
||||
assert_eq!(stats1.total_events_created, 1, "counter should reflect one recorded event");
|
||||
|
||||
// Assert — the event is retrievable by entity.
|
||||
let history = journal
|
||||
.get_entity_history("batch:42")
|
||||
.await
|
||||
.expect("history lookup");
|
||||
assert_eq!(history.len(), 1, "one event should be visible in history");
|
||||
let ev = &history[0];
|
||||
assert_eq!(ev.action, "ingest");
|
||||
assert_eq!(ev.entity_type, "test_dataset");
|
||||
assert_eq!(ev.actor, "ingest_api");
|
||||
assert!(
|
||||
ev.new_value.contains("probe.csv"),
|
||||
"new_value should carry source filename, got: {}",
|
||||
ev.new_value
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn optional_journal_field_none_is_valid_back_compat() {
|
||||
// IngestState.journal is Option<Journal>. Back-compat path: when
|
||||
// the field is None, the ingest handler MUST still succeed — the
|
||||
// journal call is fire-and-forget, never load-bearing.
|
||||
//
|
||||
// This test asserts the type shape: Option<Journal> is what we
|
||||
// expect. If a refactor makes it mandatory, this test forces an
|
||||
// explicit re-consideration.
|
||||
let none_journal: Option<Journal> = None;
|
||||
assert!(none_journal.is_none());
|
||||
|
||||
let some_journal: Option<Journal> = Some(test_journal());
|
||||
assert!(some_journal.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_record_event_fields_match_adr_012_schema() {
|
||||
// ADR-012 locks the event schema: entity_type, entity_id, field,
|
||||
// action, old_value, new_value, actor, source, workspace_id plus
|
||||
// the auto-assigned event_id + timestamp. This test pins the
|
||||
// field names so a future refactor can't silently drop one.
|
||||
let journal = test_journal();
|
||||
let base = Event {
|
||||
event_id: String::new(),
|
||||
timestamp: chrono::Utc::now(),
|
||||
entity_type: "candidate".into(),
|
||||
entity_id: "CAND-0001".into(),
|
||||
field: "phone".into(),
|
||||
action: "update".into(),
|
||||
old_value: "555-0000".into(),
|
||||
new_value: "555-9999".into(),
|
||||
actor: "recruiter".into(),
|
||||
source: "api".into(),
|
||||
workspace_id: "ws-x".into(),
|
||||
};
|
||||
journal.record(base).await.expect("record should accept full-schema event");
|
||||
let h = journal
|
||||
.get_entity_history("CAND-0001")
|
||||
.await
|
||||
.expect("lookup");
|
||||
assert_eq!(h.len(), 1);
|
||||
assert_eq!(h[0].field, "phone");
|
||||
assert_eq!(h[0].old_value, "555-0000");
|
||||
assert_eq!(h[0].new_value, "555-9999");
|
||||
assert_eq!(h[0].workspace_id, "ws-x");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
/// File watcher: monitors a directory for new files and auto-ingests them.
|
||||
/// Drop a CSV, JSON, PDF, or text file → auto-detected → Parquet → queryable.
|
||||
/// Processed files are moved to a "processed/" subdirectory.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time;
|
||||
|
||||
use catalogd::registry::Registry;
|
||||
use object_store::ObjectStore;
|
||||
|
||||
use crate::pipeline;
|
||||
|
||||
/// Watch config.
|
||||
pub struct WatchConfig {
|
||||
pub watch_dir: PathBuf,
|
||||
pub processed_dir: PathBuf,
|
||||
pub failed_dir: PathBuf,
|
||||
pub poll_interval: Duration,
|
||||
}
|
||||
|
||||
impl Default for WatchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
watch_dir: PathBuf::from("./inbox"),
|
||||
processed_dir: PathBuf::from("./inbox/processed"),
|
||||
failed_dir: PathBuf::from("./inbox/failed"),
|
||||
poll_interval: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the file watcher as a background task.
|
||||
pub fn spawn_watcher(
|
||||
config: WatchConfig,
|
||||
store: Arc<dyn ObjectStore>,
|
||||
registry: Registry,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
// Create directories
|
||||
std::fs::create_dir_all(&config.watch_dir).ok();
|
||||
std::fs::create_dir_all(&config.processed_dir).ok();
|
||||
std::fs::create_dir_all(&config.failed_dir).ok();
|
||||
|
||||
tracing::info!("file watcher started: watching {}", config.watch_dir.display());
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = time::interval(config.poll_interval);
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(e) = process_inbox(&config, &store, ®istry).await {
|
||||
tracing::error!("watcher error: {e}");
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Scan inbox directory and process any new files.
|
||||
async fn process_inbox(
|
||||
config: &WatchConfig,
|
||||
store: &Arc<dyn ObjectStore>,
|
||||
registry: &Registry,
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(&config.watch_dir)
|
||||
.map_err(|e| format!("can't read inbox: {e}"))?;
|
||||
|
||||
for entry in entries {
|
||||
let entry = match entry {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
let path = entry.path();
|
||||
|
||||
// Skip directories and hidden files. Bind filename once via
|
||||
// let-else so the subsequent use is unwrap-free — previous
|
||||
// version relied on a map_or guard above + an .unwrap() here
|
||||
// being consistent, which is a fragile invariant.
|
||||
if path.is_dir() { continue; }
|
||||
let Some(fn_os) = path.file_name() else { continue; };
|
||||
let filename = fn_os.to_string_lossy().to_string();
|
||||
if filename.starts_with('.') { continue; }
|
||||
tracing::info!("watcher: found new file '{}'", filename);
|
||||
|
||||
// Read file
|
||||
let content = match std::fs::read(&path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::error!("watcher: can't read '{}': {e}", filename);
|
||||
move_file(&path, &config.failed_dir.join(&filename));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// Ingest
|
||||
match pipeline::ingest_file(&filename, &content, None, store, registry).await {
|
||||
Ok(result) => {
|
||||
if result.deduplicated {
|
||||
tracing::info!("watcher: '{}' already ingested, skipping", filename);
|
||||
} else {
|
||||
tracing::info!("watcher: ingested '{}' → {} ({} rows, {} cols)",
|
||||
filename, result.dataset_name, result.rows, result.columns);
|
||||
}
|
||||
// Move to processed
|
||||
move_file(&path, &config.processed_dir.join(&filename));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("watcher: failed to ingest '{}': {e}", filename);
|
||||
move_file(&path, &config.failed_dir.join(&filename));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn move_file(from: &Path, to: &Path) {
|
||||
if let Err(e) = std::fs::rename(from, to) {
|
||||
// rename fails across filesystems, try copy+delete
|
||||
if let Err(e2) = std::fs::copy(from, to).and_then(|_| std::fs::remove_file(from)) {
|
||||
tracing::warn!("can't move {} → {}: {e}, copy failed: {e2}", from.display(), to.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "journald"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
storaged = { path = "../storaged" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
arrow = { workspace = true }
|
||||
parquet = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
object_store = { workspace = true }
|
||||
@@ -0,0 +1,322 @@
|
||||
/// Append-only event journal.
|
||||
/// Every mutation to any dataset is recorded here as an immutable event.
|
||||
/// Events are never modified or deleted — this IS the audit trail.
|
||||
///
|
||||
/// Storage: events buffer in memory, flush to Parquet periodically.
|
||||
/// Query: load Parquet files, filter by entity/field/actor/time.
|
||||
|
||||
use arrow::array::{ArrayRef, RecordBatch, StringArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use chrono::{DateTime, Utc};
|
||||
use object_store::ObjectStore;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use shared::arrow_helpers::record_batch_to_parquet;
|
||||
use storaged::ops;
|
||||
|
||||
/// A single mutation event. Immutable once created.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Event {
|
||||
pub event_id: String,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub entity_type: String, // "candidate", "placement", "client", etc.
|
||||
pub entity_id: String, // "CAND-00342"
|
||||
pub field: String, // "phone", "*" for insert/delete
|
||||
pub action: String, // "insert", "update", "delete"
|
||||
pub old_value: String, // "" for inserts
|
||||
pub new_value: String, // "" for deletes
|
||||
pub actor: String, // "Sarah", "ingest_pipeline", "auto_matcher"
|
||||
pub source: String, // "api", "ingest", "workspace", "agent"
|
||||
pub workspace_id: String, // "" if not workspace-scoped
|
||||
}
|
||||
|
||||
/// Arrow schema for journal Parquet files.
|
||||
fn journal_schema() -> Arc<Schema> {
|
||||
Arc::new(Schema::new(vec![
|
||||
Field::new("event_id", DataType::Utf8, false),
|
||||
Field::new("timestamp", DataType::Utf8, false),
|
||||
Field::new("entity_type", DataType::Utf8, false),
|
||||
Field::new("entity_id", DataType::Utf8, false),
|
||||
Field::new("field", DataType::Utf8, false),
|
||||
Field::new("action", DataType::Utf8, false),
|
||||
Field::new("old_value", DataType::Utf8, true),
|
||||
Field::new("new_value", DataType::Utf8, true),
|
||||
Field::new("actor", DataType::Utf8, false),
|
||||
Field::new("source", DataType::Utf8, false),
|
||||
Field::new("workspace_id", DataType::Utf8, true),
|
||||
]))
|
||||
}
|
||||
|
||||
/// The event journal — buffers events in memory, flushes to Parquet.
|
||||
#[derive(Clone)]
|
||||
pub struct Journal {
|
||||
buffer: Arc<RwLock<Vec<Event>>>,
|
||||
store: Arc<dyn ObjectStore>,
|
||||
flush_threshold: usize,
|
||||
event_counter: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl Journal {
|
||||
pub fn new(store: Arc<dyn ObjectStore>, flush_threshold: usize) -> Self {
|
||||
Self {
|
||||
buffer: Arc::new(RwLock::new(Vec::new())),
|
||||
store,
|
||||
flush_threshold,
|
||||
event_counter: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a single event. Triggers flush if buffer exceeds threshold.
|
||||
pub async fn record(&self, mut event: Event) -> Result<(), String> {
|
||||
let seq = self.event_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
event.event_id = format!("evt-{}-{seq}", event.timestamp.timestamp_millis());
|
||||
|
||||
let should_flush = {
|
||||
let mut buf = self.buffer.write().await;
|
||||
buf.push(event);
|
||||
buf.len() >= self.flush_threshold
|
||||
};
|
||||
|
||||
if should_flush {
|
||||
self.flush().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record a batch insert (multiple rows added to a dataset).
|
||||
pub async fn record_insert(
|
||||
&self,
|
||||
entity_type: &str,
|
||||
entity_id: &str,
|
||||
actor: &str,
|
||||
source: &str,
|
||||
workspace_id: &str,
|
||||
) -> Result<(), String> {
|
||||
self.record(Event {
|
||||
event_id: String::new(),
|
||||
timestamp: Utc::now(),
|
||||
entity_type: entity_type.to_string(),
|
||||
entity_id: entity_id.to_string(),
|
||||
field: "*".to_string(),
|
||||
action: "insert".to_string(),
|
||||
old_value: String::new(),
|
||||
new_value: String::new(),
|
||||
actor: actor.to_string(),
|
||||
source: source.to_string(),
|
||||
workspace_id: workspace_id.to_string(),
|
||||
}).await
|
||||
}
|
||||
|
||||
/// Record a field update.
|
||||
pub async fn record_update(
|
||||
&self,
|
||||
entity_type: &str,
|
||||
entity_id: &str,
|
||||
field: &str,
|
||||
old_value: &str,
|
||||
new_value: &str,
|
||||
actor: &str,
|
||||
source: &str,
|
||||
workspace_id: &str,
|
||||
) -> Result<(), String> {
|
||||
self.record(Event {
|
||||
event_id: String::new(),
|
||||
timestamp: Utc::now(),
|
||||
entity_type: entity_type.to_string(),
|
||||
entity_id: entity_id.to_string(),
|
||||
field: field.to_string(),
|
||||
action: "update".to_string(),
|
||||
old_value: old_value.to_string(),
|
||||
new_value: new_value.to_string(),
|
||||
actor: actor.to_string(),
|
||||
source: source.to_string(),
|
||||
workspace_id: workspace_id.to_string(),
|
||||
}).await
|
||||
}
|
||||
|
||||
/// Record a batch of events from a dataset ingest.
|
||||
pub async fn record_ingest(
|
||||
&self,
|
||||
entity_type: &str,
|
||||
count: usize,
|
||||
actor: &str,
|
||||
source_file: &str,
|
||||
) -> Result<(), String> {
|
||||
self.record(Event {
|
||||
event_id: String::new(),
|
||||
timestamp: Utc::now(),
|
||||
entity_type: entity_type.to_string(),
|
||||
entity_id: format!("batch:{count}"),
|
||||
field: "*".to_string(),
|
||||
action: "ingest".to_string(),
|
||||
old_value: String::new(),
|
||||
new_value: format!("{count} rows from {source_file}"),
|
||||
actor: actor.to_string(),
|
||||
source: "ingest".to_string(),
|
||||
workspace_id: String::new(),
|
||||
}).await
|
||||
}
|
||||
|
||||
/// Flush buffer to Parquet. Called automatically at threshold or manually.
|
||||
pub async fn flush(&self) -> Result<usize, String> {
|
||||
let events = {
|
||||
let mut buf = self.buffer.write().await;
|
||||
if buf.is_empty() { return Ok(0); }
|
||||
std::mem::take(&mut *buf)
|
||||
};
|
||||
|
||||
let n = events.len();
|
||||
let batch = events_to_batch(&events)?;
|
||||
let parquet = record_batch_to_parquet(&batch)?;
|
||||
|
||||
let ts = Utc::now().format("%Y%m%d_%H%M%S_%3f");
|
||||
let key = format!("journal/{ts}.parquet");
|
||||
ops::put(&self.store, &key, parquet).await?;
|
||||
|
||||
tracing::info!("journal: flushed {n} events to {key}");
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Query journal: get all events for a specific entity.
|
||||
pub async fn get_entity_history(&self, entity_id: &str) -> Result<Vec<Event>, String> {
|
||||
let mut all_events = Vec::new();
|
||||
|
||||
// Check in-memory buffer first
|
||||
{
|
||||
let buf = self.buffer.read().await;
|
||||
for e in buf.iter() {
|
||||
if e.entity_id == entity_id {
|
||||
all_events.push(e.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Load from persisted journal files
|
||||
let keys = ops::list(&self.store, Some("journal/")).await?;
|
||||
for key in &keys {
|
||||
if !key.ends_with(".parquet") { continue; }
|
||||
let data = ops::get(&self.store, key).await?;
|
||||
let (_, batches) = shared::arrow_helpers::parquet_to_record_batches(&data)?;
|
||||
for batch in &batches {
|
||||
let events = batch_to_events(batch)?;
|
||||
for e in events {
|
||||
if e.entity_id == entity_id {
|
||||
all_events.push(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
all_events.sort_by_key(|e| e.timestamp);
|
||||
Ok(all_events)
|
||||
}
|
||||
|
||||
/// Query journal: get recent events (last N).
|
||||
pub async fn get_recent(&self, limit: usize) -> Result<Vec<Event>, String> {
|
||||
let mut all_events = Vec::new();
|
||||
|
||||
// Buffer
|
||||
{
|
||||
let buf = self.buffer.read().await;
|
||||
all_events.extend(buf.iter().cloned());
|
||||
}
|
||||
|
||||
// Persisted (load most recent files first)
|
||||
let mut keys = ops::list(&self.store, Some("journal/")).await?;
|
||||
keys.sort();
|
||||
keys.reverse();
|
||||
|
||||
for key in &keys {
|
||||
if !key.ends_with(".parquet") { continue; }
|
||||
if all_events.len() >= limit { break; }
|
||||
let data = ops::get(&self.store, key).await?;
|
||||
let (_, batches) = shared::arrow_helpers::parquet_to_record_batches(&data)?;
|
||||
for batch in &batches {
|
||||
all_events.extend(batch_to_events(batch)?);
|
||||
}
|
||||
}
|
||||
|
||||
all_events.sort_by_key(|e| e.timestamp);
|
||||
all_events.reverse();
|
||||
all_events.truncate(limit);
|
||||
Ok(all_events)
|
||||
}
|
||||
|
||||
/// Get buffer stats.
|
||||
pub async fn stats(&self) -> JournalStats {
|
||||
let buf = self.buffer.read().await;
|
||||
let keys = ops::list(&self.store, Some("journal/")).await.unwrap_or_default();
|
||||
JournalStats {
|
||||
buffer_events: buf.len(),
|
||||
flush_threshold: self.flush_threshold,
|
||||
persisted_files: keys.len(),
|
||||
total_events_created: self.event_counter.load(std::sync::atomic::Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct JournalStats {
|
||||
pub buffer_events: usize,
|
||||
pub flush_threshold: usize,
|
||||
pub persisted_files: usize,
|
||||
pub total_events_created: u64,
|
||||
}
|
||||
|
||||
fn events_to_batch(events: &[Event]) -> Result<RecordBatch, String> {
|
||||
let schema = journal_schema();
|
||||
let arrays: Vec<ArrayRef> = vec![
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.event_id.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.timestamp.to_rfc3339()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.entity_type.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.entity_id.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.field.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.action.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.old_value.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.new_value.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.actor.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.source.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(events.iter().map(|e| e.workspace_id.as_str()).collect::<Vec<_>>())),
|
||||
];
|
||||
RecordBatch::try_new(schema, arrays).map_err(|e| format!("batch error: {e}"))
|
||||
}
|
||||
|
||||
fn batch_to_events(batch: &RecordBatch) -> Result<Vec<Event>, String> {
|
||||
let get_col = |i: usize| -> Result<&StringArray, String> {
|
||||
batch.column(i).as_any().downcast_ref::<StringArray>()
|
||||
.ok_or_else(|| format!("column {i} not string"))
|
||||
};
|
||||
|
||||
let event_ids = get_col(0)?;
|
||||
let timestamps = get_col(1)?;
|
||||
let entity_types = get_col(2)?;
|
||||
let entity_ids = get_col(3)?;
|
||||
let fields = get_col(4)?;
|
||||
let actions = get_col(5)?;
|
||||
let old_values = get_col(6)?;
|
||||
let new_values = get_col(7)?;
|
||||
let actors = get_col(8)?;
|
||||
let sources = get_col(9)?;
|
||||
let workspace_ids = get_col(10)?;
|
||||
|
||||
let mut events = Vec::with_capacity(batch.num_rows());
|
||||
for i in 0..batch.num_rows() {
|
||||
events.push(Event {
|
||||
event_id: event_ids.value(i).to_string(),
|
||||
timestamp: timestamps.value(i).parse().unwrap_or_else(|_| Utc::now()),
|
||||
entity_type: entity_types.value(i).to_string(),
|
||||
entity_id: entity_ids.value(i).to_string(),
|
||||
field: fields.value(i).to_string(),
|
||||
action: actions.value(i).to_string(),
|
||||
old_value: old_values.value(i).to_string(),
|
||||
new_value: new_values.value(i).to_string(),
|
||||
actor: actors.value(i).to_string(),
|
||||
source: sources.value(i).to_string(),
|
||||
workspace_id: workspace_ids.value(i).to_string(),
|
||||
});
|
||||
}
|
||||
Ok(events)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod journal;
|
||||
pub mod service;
|
||||
@@ -0,0 +1,131 @@
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::journal::{Event, Journal};
|
||||
|
||||
pub fn router(journal: Journal) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/event", post(record_event))
|
||||
.route("/update", post(record_update))
|
||||
.route("/flush", post(flush))
|
||||
.route("/history/{entity_id}", get(entity_history))
|
||||
.route("/recent", get(recent_events))
|
||||
.route("/stats", get(stats))
|
||||
.with_state(journal)
|
||||
}
|
||||
|
||||
async fn health() -> &'static str {
|
||||
"journald ok"
|
||||
}
|
||||
|
||||
// --- Record events ---
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RecordEventRequest {
|
||||
entity_type: String,
|
||||
entity_id: String,
|
||||
action: String,
|
||||
field: Option<String>,
|
||||
old_value: Option<String>,
|
||||
new_value: Option<String>,
|
||||
actor: String,
|
||||
source: String,
|
||||
workspace_id: Option<String>,
|
||||
}
|
||||
|
||||
async fn record_event(
|
||||
State(journal): State<Journal>,
|
||||
Json(req): Json<RecordEventRequest>,
|
||||
) -> impl IntoResponse {
|
||||
let event = Event {
|
||||
event_id: String::new(), // assigned by journal
|
||||
timestamp: chrono::Utc::now(),
|
||||
entity_type: req.entity_type,
|
||||
entity_id: req.entity_id,
|
||||
field: req.field.unwrap_or_else(|| "*".to_string()),
|
||||
action: req.action,
|
||||
old_value: req.old_value.unwrap_or_default(),
|
||||
new_value: req.new_value.unwrap_or_default(),
|
||||
actor: req.actor,
|
||||
source: req.source,
|
||||
workspace_id: req.workspace_id.unwrap_or_default(),
|
||||
};
|
||||
|
||||
match journal.record(event).await {
|
||||
Ok(()) => Ok((StatusCode::CREATED, "event recorded")),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct UpdateRequest {
|
||||
entity_type: String,
|
||||
entity_id: String,
|
||||
field: String,
|
||||
old_value: String,
|
||||
new_value: String,
|
||||
actor: String,
|
||||
workspace_id: Option<String>,
|
||||
}
|
||||
|
||||
async fn record_update(
|
||||
State(journal): State<Journal>,
|
||||
Json(req): Json<UpdateRequest>,
|
||||
) -> impl IntoResponse {
|
||||
match journal.record_update(
|
||||
&req.entity_type, &req.entity_id, &req.field,
|
||||
&req.old_value, &req.new_value, &req.actor, "api",
|
||||
&req.workspace_id.unwrap_or_default(),
|
||||
).await {
|
||||
Ok(()) => Ok((StatusCode::CREATED, "update recorded")),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Flush ---
|
||||
|
||||
async fn flush(State(journal): State<Journal>) -> impl IntoResponse {
|
||||
match journal.flush().await {
|
||||
Ok(n) => Ok(format!("flushed {n} events")),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Query ---
|
||||
|
||||
async fn entity_history(
|
||||
State(journal): State<Journal>,
|
||||
Path(entity_id): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match journal.get_entity_history(&entity_id).await {
|
||||
Ok(events) => Ok(Json(events)),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RecentQuery {
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
async fn recent_events(
|
||||
State(journal): State<Journal>,
|
||||
Query(q): Query<RecentQuery>,
|
||||
) -> impl IntoResponse {
|
||||
let limit = q.limit.unwrap_or(50);
|
||||
match journal.get_recent(limit).await {
|
||||
Ok(events) => Ok(Json(events)),
|
||||
Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn stats(State(journal): State<Journal>) -> impl IntoResponse {
|
||||
Json(journal.stats().await)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
[package]
|
||||
name = "lance-bench"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
# Standalone pilot for Phase B (see docs/EXECUTION_PLAN.md).
|
||||
# Deliberately NOT sharing workspace deps — Lance 4.x pulls in its own
|
||||
# DataFusion and Arrow versions incompatible with the rest of the stack.
|
||||
# Isolating the pilot means we don't force a workspace-wide upgrade until
|
||||
# we've decided Lance is worth it.
|
||||
|
||||
[dependencies]
|
||||
# Only the features we actually need — the default brings in AWS/Azure/GCP/HF etc
|
||||
# which is ~200 extra crates we don't care about for a local pilot.
|
||||
lance = { version = "4.0", default-features = false }
|
||||
# Lance exposes DatasetIndexExt, IndexType, and IvfBuildParams through
|
||||
# its sub-crates which must be imported directly — lance itself doesn't
|
||||
# re-export them at a convenient path.
|
||||
lance-index = { version = "4.0", default-features = false }
|
||||
lance-linalg = { version = "4.0", default-features = false }
|
||||
|
||||
# Arrow re-exported by Lance; pin to a range Lance picks so types match.
|
||||
arrow = "57"
|
||||
arrow-array = "57"
|
||||
arrow-schema = "57"
|
||||
|
||||
# Also need to read the EXISTING Parquet vector files so we can compare.
|
||||
# These live in data/vectors/*.parquet. Lance's internal Parquet reading
|
||||
# might differ from ours; using our format's Arrow/Parquet versions for
|
||||
# the read side keeps the inputs identical.
|
||||
parquet = "57"
|
||||
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
futures = "0.3"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
anyhow = "1"
|
||||
bytes = "1"
|
||||
|
||||
[[bin]]
|
||||
name = "lance-bench"
|
||||
path = "src/main.rs"
|
||||
@@ -0,0 +1,633 @@
|
||||
//! Phase B: Lance pilot benchmark.
|
||||
//!
|
||||
//! Standalone binary that compares Lance vector storage against our
|
||||
//! Parquet-with-binary-blob + in-RAM HNSW approach. See
|
||||
//! docs/EXECUTION_PLAN.md for the decision rules this fuels.
|
||||
//!
|
||||
//! Inputs:
|
||||
//! data/vectors/resumes_100k_v2.parquet — existing 100K × 768d embeddings
|
||||
//!
|
||||
//! Output:
|
||||
//! A JSON report printed to stdout with measurements for:
|
||||
//! - Cold load time (parquet → arrow) vs Lance open + scan
|
||||
//! - Disk size
|
||||
//! - Vector search latency (p50 / p95 / p99)
|
||||
//! - Single-row random access
|
||||
//! - Append cost (adding 10K rows)
|
||||
//!
|
||||
//! Usage:
|
||||
//! cargo run --bin lance-bench -- \
|
||||
//! --parquet data/vectors/resumes_100k_v2.parquet \
|
||||
//! --lance-out /tmp/lance_resumes_100k_v2 \
|
||||
//! --json-out /tmp/lance_bench.json
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use arrow_array::{Array, ArrayRef, BinaryArray, FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct BenchReport {
|
||||
vectors: usize,
|
||||
dimensions: usize,
|
||||
parquet_path: String,
|
||||
lance_path: String,
|
||||
|
||||
// Parquet baseline
|
||||
parquet_disk_bytes: u64,
|
||||
parquet_cold_load_secs: f32,
|
||||
|
||||
// Lance numbers
|
||||
lance_write_secs: f32,
|
||||
lance_disk_bytes: u64,
|
||||
lance_cold_open_secs: f32,
|
||||
|
||||
// Index + search
|
||||
lance_index_build_secs: Option<f32>,
|
||||
lance_index_disk_bytes: Option<u64>,
|
||||
lance_search_p50_us: Option<f32>,
|
||||
lance_search_p95_us: Option<f32>,
|
||||
lance_search_p99_us: Option<f32>,
|
||||
|
||||
// Architectural features Parquet+sidecar can't cheaply do
|
||||
lance_random_row_access_us: Option<f32>, // fetch one row by row_id
|
||||
parquet_random_row_access_us: Option<f32>, // for comparison — full scan cost
|
||||
lance_append_10k_secs: Option<f32>, // add 10K new rows
|
||||
lance_append_disk_bytes_added: Option<u64>,
|
||||
|
||||
// Head-to-head reference (from our own measurements)
|
||||
reference_hnsw_p50_us: f32,
|
||||
reference_hnsw_p95_us: f32,
|
||||
reference_brute_force_us: f32,
|
||||
reference_hnsw_build_secs: f32,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// Simple positional args: parquet_in, lance_out.
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let parquet_path = args
|
||||
.get(1)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "data/vectors/resumes_100k_v2.parquet".to_string());
|
||||
let lance_path = args
|
||||
.get(2)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "/tmp/lance_bench_dataset".to_string());
|
||||
|
||||
eprintln!("=== Phase B Lance pilot ===");
|
||||
eprintln!("input parquet: {}", parquet_path);
|
||||
eprintln!("output lance: {}", lance_path);
|
||||
|
||||
// --- 1. Cold-load the existing Parquet vector index into memory
|
||||
eprintln!("\n[1/4] reading Parquet baseline...");
|
||||
let t0 = Instant::now();
|
||||
let (schema, batches, total_rows) = read_parquet_vectors(&parquet_path)
|
||||
.context("read parquet")?;
|
||||
let parquet_cold_load_secs = t0.elapsed().as_secs_f32();
|
||||
let parquet_disk_bytes = std::fs::metadata(&parquet_path)?.len();
|
||||
|
||||
let dims = detect_vector_dims(&batches)?;
|
||||
eprintln!(
|
||||
" loaded {} rows, {} columns, vectors={}d, disk={:.1} MB, cold load={:.2}s",
|
||||
total_rows,
|
||||
schema.fields().len(),
|
||||
dims,
|
||||
parquet_disk_bytes as f64 / 1_000_000.0,
|
||||
parquet_cold_load_secs,
|
||||
);
|
||||
|
||||
// --- 2. Convert from binary-blob-of-f32 to Lance's FixedSizeList<Float32>
|
||||
eprintln!("\n[2/4] converting binary-blob vectors to Arrow FixedSizeList...");
|
||||
let t0 = Instant::now();
|
||||
let (lance_schema, lance_batches) = convert_to_fixed_size_list(&schema, batches, dims)?;
|
||||
eprintln!(" conversion took {:.2}s", t0.elapsed().as_secs_f32());
|
||||
|
||||
// --- 3. Write as Lance dataset
|
||||
eprintln!("\n[3/4] writing Lance dataset...");
|
||||
let t0 = Instant::now();
|
||||
// Clean up any prior run
|
||||
let _ = std::fs::remove_dir_all(&lance_path);
|
||||
write_lance_dataset(&lance_path, lance_schema.clone(), lance_batches).await?;
|
||||
let lance_write_secs = t0.elapsed().as_secs_f32();
|
||||
let lance_disk_bytes = dir_size_bytes(&lance_path);
|
||||
eprintln!(
|
||||
" write took {:.2}s, disk={:.1} MB",
|
||||
lance_write_secs,
|
||||
lance_disk_bytes as f64 / 1_000_000.0,
|
||||
);
|
||||
|
||||
// --- 4. Cold open + scan the Lance dataset
|
||||
eprintln!("\n[4/6] cold-opening Lance dataset...");
|
||||
let t0 = Instant::now();
|
||||
let scanned_rows = cold_open_and_scan_lance(&lance_path).await?;
|
||||
let lance_cold_open_secs = t0.elapsed().as_secs_f32();
|
||||
eprintln!(
|
||||
" open + full scan: {} rows in {:.2}s",
|
||||
scanned_rows, lance_cold_open_secs,
|
||||
);
|
||||
|
||||
// --- 5. Build a vector index on the Lance dataset
|
||||
eprintln!("\n[5/6] building Lance vector index (IVF_PQ)...");
|
||||
let t0 = Instant::now();
|
||||
let index_built = build_lance_vector_index(&lance_path, dims).await;
|
||||
let (lance_index_build_secs, lance_index_disk_bytes) = match index_built {
|
||||
Ok(()) => {
|
||||
let secs = t0.elapsed().as_secs_f32();
|
||||
let disk = dir_size_bytes(&lance_path) - lance_disk_bytes;
|
||||
eprintln!(" built in {:.2}s, index adds {:.1} MB on disk", secs, disk as f64 / 1e6);
|
||||
(Some(secs), Some(disk))
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" index build failed: {e:#}");
|
||||
(None, None)
|
||||
}
|
||||
};
|
||||
|
||||
// --- 6. Run search queries, measure latency
|
||||
eprintln!("\n[6/6] running vector search benchmarks...");
|
||||
let search_stats = if lance_index_build_secs.is_some() {
|
||||
run_search_benchmarks(&lance_path, dims).await.ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (lance_search_p50, lance_search_p95, lance_search_p99) = match search_stats {
|
||||
Some((p50, p95, p99)) => {
|
||||
eprintln!(" p50={:.0}us p95={:.0}us p99={:.0}us", p50, p95, p99);
|
||||
(Some(p50), Some(p95), Some(p99))
|
||||
}
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
// --- Random access comparison
|
||||
eprintln!("\n[7/8] random row access — Lance vs full-scan Parquet...");
|
||||
let lance_random = measure_random_access_lance(&lance_path).await.ok();
|
||||
let parquet_random = measure_random_access_parquet(&parquet_path).ok();
|
||||
if let Some(us) = lance_random {
|
||||
eprintln!(" Lance random-fetch avg: {:.0}us", us);
|
||||
}
|
||||
if let Some(us) = parquet_random {
|
||||
eprintln!(" Parquet full-scan-to-row avg: {:.0}us", us);
|
||||
}
|
||||
|
||||
// --- Append cost
|
||||
eprintln!("\n[8/8] append 10K new rows to existing dataset...");
|
||||
let t0 = Instant::now();
|
||||
let pre_append_bytes = dir_size_bytes(&lance_path);
|
||||
let append_result = append_10k_rows(&lance_path, dims).await;
|
||||
let (lance_append_secs, lance_append_bytes) = match append_result {
|
||||
Ok(()) => {
|
||||
let secs = t0.elapsed().as_secs_f32();
|
||||
let bytes = dir_size_bytes(&lance_path).saturating_sub(pre_append_bytes);
|
||||
eprintln!(" append took {:.2}s, added {:.1} MB", secs, bytes as f64 / 1e6);
|
||||
(Some(secs), Some(bytes))
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" append failed: {e:#}");
|
||||
(None, None)
|
||||
}
|
||||
};
|
||||
|
||||
// --- Report
|
||||
let report = BenchReport {
|
||||
vectors: total_rows,
|
||||
dimensions: dims,
|
||||
parquet_path: parquet_path.clone(),
|
||||
lance_path: lance_path.clone(),
|
||||
parquet_disk_bytes,
|
||||
parquet_cold_load_secs,
|
||||
lance_write_secs,
|
||||
lance_disk_bytes,
|
||||
lance_cold_open_secs,
|
||||
lance_index_build_secs,
|
||||
lance_index_disk_bytes,
|
||||
lance_search_p50_us: lance_search_p50,
|
||||
lance_search_p95_us: lance_search_p95,
|
||||
lance_search_p99_us: lance_search_p99,
|
||||
lance_random_row_access_us: lance_random,
|
||||
parquet_random_row_access_us: parquet_random,
|
||||
lance_append_10k_secs: lance_append_secs,
|
||||
lance_append_disk_bytes_added: lance_append_bytes,
|
||||
// From our Phase 15 trial on the SAME index (ec=80 es=30, recall=1.00):
|
||||
reference_hnsw_p50_us: 873.0,
|
||||
reference_hnsw_p95_us: 1413.0,
|
||||
reference_brute_force_us: 43983.0,
|
||||
reference_hnsw_build_secs: 230.0,
|
||||
};
|
||||
|
||||
let json = serde_json::to_string_pretty(&report)?;
|
||||
println!("{}", json);
|
||||
|
||||
eprintln!("\n=== Summary ===");
|
||||
eprintln!(" Parquet cold load: {:.2}s", report.parquet_cold_load_secs);
|
||||
eprintln!(" Lance cold open: {:.2}s ({})",
|
||||
report.lance_cold_open_secs,
|
||||
format_ratio(report.parquet_cold_load_secs, report.lance_cold_open_secs));
|
||||
eprintln!(" Parquet disk: {:.1} MB", report.parquet_disk_bytes as f64 / 1e6);
|
||||
eprintln!(" Lance disk: {:.1} MB ({})",
|
||||
report.lance_disk_bytes as f64 / 1e6,
|
||||
format_ratio(report.parquet_disk_bytes as f32, report.lance_disk_bytes as f32));
|
||||
if let (Some(p50), Some(p95)) = (report.lance_search_p50_us, report.lance_search_p95_us) {
|
||||
eprintln!(" Lance search p50: {:.0}us vs our HNSW {:.0}us ({})",
|
||||
p50, report.reference_hnsw_p50_us,
|
||||
format_ratio(report.reference_hnsw_p50_us, p50));
|
||||
eprintln!(" Lance search p95: {:.0}us vs our HNSW {:.0}us ({})",
|
||||
p95, report.reference_hnsw_p95_us,
|
||||
format_ratio(report.reference_hnsw_p95_us, p95));
|
||||
eprintln!(" Speedup vs brute force: {:.1}× (Lance) vs {:.1}× (HNSW)",
|
||||
report.reference_brute_force_us / p50,
|
||||
report.reference_brute_force_us / report.reference_hnsw_p50_us);
|
||||
}
|
||||
if let Some(build) = report.lance_index_build_secs {
|
||||
eprintln!(" Index build: {:.1}s (Lance IVF_PQ) vs {:.0}s (our HNSW ec=80) ({}× faster)",
|
||||
build, report.reference_hnsw_build_secs, report.reference_hnsw_build_secs / build);
|
||||
}
|
||||
if let (Some(lance_us), Some(parquet_us)) = (report.lance_random_row_access_us, report.parquet_random_row_access_us) {
|
||||
eprintln!(" Random row access: {:.0}us (Lance) vs {:.0}us (Parquet scan) ({})",
|
||||
lance_us, parquet_us, format_ratio(parquet_us, lance_us));
|
||||
}
|
||||
if let Some(append_secs) = report.lance_append_10k_secs {
|
||||
eprintln!(" Append 10K rows: {:.2}s (Lance native) [Parquet would require full rewrite]",
|
||||
append_secs);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn format_ratio(baseline: f32, candidate: f32) -> String {
|
||||
if candidate == 0.0 { return "inf".into(); }
|
||||
let ratio = baseline / candidate;
|
||||
if ratio >= 1.0 {
|
||||
format!("{:.2}× faster/smaller", ratio)
|
||||
} else {
|
||||
format!("{:.2}× slower/larger", 1.0 / ratio)
|
||||
}
|
||||
}
|
||||
|
||||
fn dir_size_bytes(path: &str) -> u64 {
|
||||
fn recurse(p: &std::path::Path) -> u64 {
|
||||
let Ok(meta) = std::fs::metadata(p) else { return 0; };
|
||||
if meta.is_file() { return meta.len(); }
|
||||
let Ok(entries) = std::fs::read_dir(p) else { return 0; };
|
||||
entries
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| recurse(&e.path()))
|
||||
.sum()
|
||||
}
|
||||
recurse(std::path::Path::new(path))
|
||||
}
|
||||
|
||||
/// Read the existing vector Parquet (binary-blob format: source, doc_id,
|
||||
/// chunk_idx, chunk_text, vector as Binary bytes).
|
||||
fn read_parquet_vectors(path: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>, usize)> {
|
||||
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
|
||||
use std::fs::File;
|
||||
|
||||
let file = File::open(path).with_context(|| format!("open {path}"))?;
|
||||
let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
|
||||
let schema = builder.schema().clone();
|
||||
let reader = builder.build()?;
|
||||
let batches: Vec<RecordBatch> = reader.collect::<Result<Vec<_>, _>>()?;
|
||||
let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
|
||||
Ok((schema, batches, rows))
|
||||
}
|
||||
|
||||
fn detect_vector_dims(batches: &[RecordBatch]) -> Result<usize> {
|
||||
for batch in batches {
|
||||
let vector_col_idx = batch
|
||||
.schema()
|
||||
.index_of("vector")
|
||||
.context("no 'vector' column in parquet")?;
|
||||
let col = batch.column(vector_col_idx);
|
||||
if let Some(binary) = col.as_any().downcast_ref::<BinaryArray>() {
|
||||
for i in 0..binary.len() {
|
||||
if !binary.is_null(i) {
|
||||
let bytes = binary.value(i);
|
||||
return Ok(bytes.len() / 4); // f32 = 4 bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
anyhow::bail!("could not determine vector dimensions")
|
||||
}
|
||||
|
||||
/// Convert our binary-blob vector representation into Arrow's native
|
||||
/// FixedSizeList<Float32> — that's what Lance expects for vector columns.
|
||||
fn convert_to_fixed_size_list(
|
||||
schema: &Arc<Schema>,
|
||||
batches: Vec<RecordBatch>,
|
||||
dims: usize,
|
||||
) -> Result<(Arc<Schema>, Vec<RecordBatch>)> {
|
||||
// New schema keeps everything identical but replaces the vector column
|
||||
// with a FixedSizeList<Float32, dims>.
|
||||
let new_fields: Vec<Arc<Field>> = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|f| {
|
||||
if f.name() == "vector" {
|
||||
Arc::new(Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(
|
||||
Arc::new(Field::new("item", DataType::Float32, true)),
|
||||
dims as i32,
|
||||
),
|
||||
false,
|
||||
))
|
||||
} else {
|
||||
f.clone()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let new_schema = Arc::new(Schema::new(new_fields));
|
||||
|
||||
let mut new_batches = Vec::with_capacity(batches.len());
|
||||
for batch in batches {
|
||||
let vector_idx = batch.schema().index_of("vector")?;
|
||||
let mut new_arrays: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());
|
||||
for (i, col) in batch.columns().iter().enumerate() {
|
||||
if i == vector_idx {
|
||||
let binary = col
|
||||
.as_any()
|
||||
.downcast_ref::<BinaryArray>()
|
||||
.context("vector column must be Binary")?;
|
||||
let fsl = binary_to_fixed_size_list(binary, dims)?;
|
||||
new_arrays.push(Arc::new(fsl));
|
||||
} else {
|
||||
new_arrays.push(col.clone());
|
||||
}
|
||||
}
|
||||
new_batches.push(RecordBatch::try_new(new_schema.clone(), new_arrays)?);
|
||||
}
|
||||
|
||||
Ok((new_schema, new_batches))
|
||||
}
|
||||
|
||||
fn binary_to_fixed_size_list(binary: &BinaryArray, dims: usize) -> Result<FixedSizeListArray> {
|
||||
let n = binary.len();
|
||||
let mut all_floats: Vec<f32> = Vec::with_capacity(n * dims);
|
||||
for i in 0..n {
|
||||
if binary.is_null(i) {
|
||||
all_floats.extend(std::iter::repeat(0.0).take(dims));
|
||||
continue;
|
||||
}
|
||||
let bytes = binary.value(i);
|
||||
if bytes.len() != dims * 4 {
|
||||
anyhow::bail!(
|
||||
"row {} has {} bytes, expected {} ({} × f32)",
|
||||
i, bytes.len(), dims * 4, dims,
|
||||
);
|
||||
}
|
||||
for chunk in bytes.chunks_exact(4) {
|
||||
all_floats.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]));
|
||||
}
|
||||
}
|
||||
let values = Float32Array::from(all_floats);
|
||||
let field = Arc::new(Field::new("item", DataType::Float32, true));
|
||||
FixedSizeListArray::try_new(field, dims as i32, Arc::new(values), None)
|
||||
.context("build FixedSizeListArray")
|
||||
}
|
||||
|
||||
/// Write batches into a Lance dataset at the given path.
|
||||
async fn write_lance_dataset(
|
||||
path: &str,
|
||||
schema: Arc<Schema>,
|
||||
batches: Vec<RecordBatch>,
|
||||
) -> Result<()> {
|
||||
use lance::dataset::{Dataset, WriteParams};
|
||||
|
||||
let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
|
||||
Dataset::write(reader, path, Some(WriteParams::default()))
|
||||
.await
|
||||
.context("Dataset::write")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open a Lance dataset cold (from disk) and scan it fully — measuring the
|
||||
/// equivalent of our "load embeddings from Parquet" cost.
|
||||
async fn cold_open_and_scan_lance(path: &str) -> Result<usize> {
|
||||
use futures::StreamExt;
|
||||
use lance::dataset::Dataset;
|
||||
|
||||
let dataset = Dataset::open(path).await.context("Dataset::open")?;
|
||||
let scanner = dataset.scan();
|
||||
let mut stream = scanner.try_into_stream().await?;
|
||||
let mut total = 0usize;
|
||||
while let Some(batch) = stream.next().await {
|
||||
let batch = batch?;
|
||||
total += batch.num_rows();
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Build an IVF_PQ vector index on the `vector` column. IVF_PQ (Inverted File
|
||||
/// with Product Quantization) is Lance's native ANN index — comparable to
|
||||
/// HNSW in intent, but on-disk and compatible with Lance's random-access
|
||||
/// model.
|
||||
async fn build_lance_vector_index(path: &str, _dims: usize) -> Result<()> {
|
||||
use lance::dataset::Dataset;
|
||||
use lance::index::vector::VectorIndexParams;
|
||||
use lance_index::{DatasetIndexExt, IndexType};
|
||||
use lance_linalg::distance::MetricType;
|
||||
|
||||
let mut dataset = Dataset::open(path).await?;
|
||||
|
||||
// IVF_PQ with ~sqrt(N) partitions is a reasonable default for 100K.
|
||||
// num_sub_vectors must divide dims evenly: 768/48 = 16 dims per subvector.
|
||||
// num_bits = 8 gives 256 codes per subvector (good recall/size trade).
|
||||
// max_iterations = 50 is plenty for this scale.
|
||||
let params = VectorIndexParams::ivf_pq(
|
||||
316, // num_partitions (~sqrt(100000))
|
||||
8, // num_bits
|
||||
48, // num_sub_vectors
|
||||
MetricType::Cosine,
|
||||
50, // max_iterations
|
||||
);
|
||||
|
||||
dataset
|
||||
.create_index(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
Some("vec_idx".into()),
|
||||
¶ms,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.context("create_index")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run N vector searches against the Lance dataset and return (p50, p95, p99) latencies in us.
|
||||
/// Uses a handful of random rows as queries — same pattern as our harness::synthetic_from_chunks.
|
||||
async fn run_search_benchmarks(path: &str, _dims: usize) -> Result<(f32, f32, f32)> {
|
||||
use futures::StreamExt;
|
||||
use lance::dataset::Dataset;
|
||||
|
||||
let dataset = Dataset::open(path).await?;
|
||||
|
||||
// Pick 20 representative query vectors from the data itself.
|
||||
// (Synthetic — same pattern as our existing harness.)
|
||||
let query_vectors = sample_query_vectors(&dataset, 20).await?;
|
||||
|
||||
let mut latencies_us: Vec<f32> = Vec::with_capacity(query_vectors.len());
|
||||
for (i, qv) in query_vectors.iter().enumerate() {
|
||||
let qarr = Arc::new(Float32Array::from(qv.clone())) as ArrayRef;
|
||||
|
||||
let t0 = Instant::now();
|
||||
let mut scanner = dataset.scan();
|
||||
scanner
|
||||
.nearest("vector", qarr.as_any().downcast_ref::<Float32Array>().unwrap(), 10)
|
||||
.context("scanner.nearest")?;
|
||||
let mut stream = scanner.try_into_stream().await?;
|
||||
let mut hits = 0;
|
||||
while let Some(batch) = stream.next().await {
|
||||
let batch = batch?;
|
||||
hits += batch.num_rows();
|
||||
}
|
||||
let us = t0.elapsed().as_micros() as f32;
|
||||
latencies_us.push(us);
|
||||
if i == 0 {
|
||||
eprintln!(" first query: {} hits in {:.0}us (includes any lazy init)", hits, us);
|
||||
}
|
||||
}
|
||||
|
||||
latencies_us.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let p = |pct: f32| -> f32 {
|
||||
let idx = ((latencies_us.len() as f32 - 1.0) * pct).round() as usize;
|
||||
latencies_us[idx.min(latencies_us.len() - 1)]
|
||||
};
|
||||
Ok((p(0.50), p(0.95), p(0.99)))
|
||||
}
|
||||
|
||||
/// Random row access via Lance's `take` — fetch 20 random rows by index, measure avg latency.
|
||||
async fn measure_random_access_lance(path: &str) -> Result<f32> {
|
||||
use lance::dataset::Dataset;
|
||||
let dataset = Dataset::open(path).await?;
|
||||
let n = dataset.count_rows(None).await?;
|
||||
let indices: Vec<u64> = (0..20).map(|i| ((i as u64) * (n as u64 / 23)) % (n as u64)).collect();
|
||||
|
||||
// Full-schema projection — Lance's Schema implements Into<ProjectionRequest>.
|
||||
let schema = dataset.schema().clone();
|
||||
let mut total_us: u128 = 0;
|
||||
for idx in &indices {
|
||||
let t0 = Instant::now();
|
||||
let _batch = dataset.take(&[*idx], schema.clone()).await?;
|
||||
total_us += t0.elapsed().as_micros();
|
||||
}
|
||||
Ok(total_us as f32 / indices.len() as f32)
|
||||
}
|
||||
|
||||
/// Random row access for Parquet — full scan + filter. There's no random-access
|
||||
/// primitive in vanilla Parquet, so this is the cost of finding one specific row.
|
||||
/// This is the cost our current design pays for "get doc X's full text for RAG."
|
||||
fn measure_random_access_parquet(path: &str) -> Result<f32> {
|
||||
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
|
||||
use std::fs::File;
|
||||
|
||||
// We simulate 5 lookups — full scan each time. 20 would be painful.
|
||||
let iters = 5;
|
||||
let mut total_us: u128 = 0;
|
||||
for _ in 0..iters {
|
||||
let t0 = Instant::now();
|
||||
let file = File::open(path)?;
|
||||
let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
|
||||
let reader = builder.build()?;
|
||||
// Iterate until we've conceptually found a row — we stop early if
|
||||
// we wanted row 50000, but we have to at least read its batch.
|
||||
let mut seen = 0usize;
|
||||
for b in reader {
|
||||
let b = b?;
|
||||
seen += b.num_rows();
|
||||
if seen > 50000 { break; }
|
||||
}
|
||||
total_us += t0.elapsed().as_micros();
|
||||
}
|
||||
Ok(total_us as f32 / iters as f32)
|
||||
}
|
||||
|
||||
/// Append 10K new rows to the existing Lance dataset.
|
||||
/// Measures the "ingest delta" cost without full rewrite.
|
||||
async fn append_10k_rows(path: &str, dims: usize) -> Result<()> {
|
||||
use lance::dataset::{Dataset, WriteMode, WriteParams};
|
||||
|
||||
let dataset = Dataset::open(path).await?;
|
||||
let schema = dataset.schema();
|
||||
let arrow_schema: Arc<Schema> = Arc::new(schema.into());
|
||||
|
||||
// Build a 10K row batch with random-ish data matching the existing schema.
|
||||
let n = 10_000;
|
||||
let arrays: Vec<ArrayRef> = arrow_schema
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|f| -> Result<ArrayRef> {
|
||||
match f.data_type() {
|
||||
DataType::Utf8 => {
|
||||
let vals: Vec<String> = (0..n).map(|i| format!("appended-{}", i)).collect();
|
||||
Ok(Arc::new(arrow_array::StringArray::from(vals)))
|
||||
}
|
||||
DataType::Int32 => {
|
||||
let vals: Vec<i32> = (0..n as i32).collect();
|
||||
Ok(Arc::new(arrow_array::Int32Array::from(vals)))
|
||||
}
|
||||
DataType::FixedSizeList(_, _) => {
|
||||
let floats: Vec<f32> = (0..n * dims).map(|i| (i as f32).sin()).collect();
|
||||
let values = Float32Array::from(floats);
|
||||
let field = Arc::new(Field::new("item", DataType::Float32, true));
|
||||
let fsl = FixedSizeListArray::try_new(field, dims as i32, Arc::new(values), None)?;
|
||||
Ok(Arc::new(fsl))
|
||||
}
|
||||
other => anyhow::bail!("unsupported append column type: {:?}", other),
|
||||
}
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
let batch = RecordBatch::try_new(arrow_schema.clone(), arrays)?;
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), arrow_schema);
|
||||
let params = WriteParams { mode: WriteMode::Append, ..Default::default() };
|
||||
Dataset::write(reader, path, Some(params)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Grab a few existing vectors from the dataset to use as self-similar queries.
|
||||
async fn sample_query_vectors(
|
||||
dataset: &lance::dataset::Dataset,
|
||||
count: usize,
|
||||
) -> Result<Vec<Vec<f32>>> {
|
||||
use futures::StreamExt;
|
||||
|
||||
// Just take the first `count` rows; good enough for latency measurement.
|
||||
let scanner = dataset.scan();
|
||||
let mut scanner = scanner;
|
||||
scanner.limit(Some(count as i64), None)?;
|
||||
scanner.project(&["vector"])?;
|
||||
let mut stream = scanner.try_into_stream().await?;
|
||||
|
||||
let mut out = Vec::with_capacity(count);
|
||||
while let Some(batch) = stream.next().await {
|
||||
let batch = batch?;
|
||||
let vector_col = batch
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<FixedSizeListArray>()
|
||||
.context("vector column must be FixedSizeList")?;
|
||||
|
||||
for row in 0..vector_col.len() {
|
||||
if out.len() >= count { break; }
|
||||
let values = vector_col.value(row);
|
||||
let f32_arr = values
|
||||
.as_any()
|
||||
.downcast_ref::<Float32Array>()
|
||||
.context("inner array must be Float32")?;
|
||||
let mut v = Vec::with_capacity(f32_arr.len());
|
||||
for i in 0..f32_arr.len() {
|
||||
v.push(f32_arr.value(i));
|
||||
}
|
||||
out.push(v);
|
||||
}
|
||||
if out.len() >= count { break; }
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
[package]
|
||||
name = "proto"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
tonic = { workspace = true }
|
||||
prost = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build = { workspace = true }
|
||||
@@ -0,0 +1,4 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tonic_build::compile_protos("../../proto/lakehouse.proto")?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod lakehouse {
|
||||
tonic::include_proto!("lakehouse");
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "queryd"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
catalogd = { path = "../catalogd" }
|
||||
storaged = { path = "../storaged" }
|
||||
truth = { path = "../truth" }
|
||||
tokio = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
datafusion = { workspace = true }
|
||||
object_store = { workspace = true }
|
||||
arrow = { workspace = true }
|
||||
parquet = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
url = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
@@ -0,0 +1,135 @@
|
||||
/// In-memory cache for hot datasets.
|
||||
/// Pinned datasets are loaded as Arrow RecordBatches and served from RAM.
|
||||
/// LRU eviction keeps memory bounded.
|
||||
|
||||
use arrow::array::RecordBatch;
|
||||
use arrow::datatypes::SchemaRef;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// A cached dataset — schema + all batches in memory.
|
||||
#[derive(Clone)]
|
||||
pub struct CachedDataset {
|
||||
pub name: String,
|
||||
pub schema: SchemaRef,
|
||||
pub batches: Vec<RecordBatch>,
|
||||
pub row_count: usize,
|
||||
pub size_bytes: usize, // approximate
|
||||
pub last_accessed: std::time::Instant,
|
||||
}
|
||||
|
||||
/// Memory cache with LRU eviction.
|
||||
#[derive(Clone)]
|
||||
pub struct MemCache {
|
||||
datasets: Arc<RwLock<HashMap<String, CachedDataset>>>,
|
||||
max_bytes: usize,
|
||||
}
|
||||
|
||||
impl MemCache {
|
||||
pub fn new(max_bytes: usize) -> Self {
|
||||
Self {
|
||||
datasets: Arc::new(RwLock::new(HashMap::new())),
|
||||
max_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Load a dataset into the cache.
|
||||
pub async fn put(&self, name: &str, schema: SchemaRef, batches: Vec<RecordBatch>) {
|
||||
let row_count: usize = batches.iter().map(|b| b.num_rows()).sum();
|
||||
let size_bytes: usize = batches.iter()
|
||||
.map(|b| b.get_array_memory_size())
|
||||
.sum();
|
||||
|
||||
// Evict if needed
|
||||
self.evict_to_fit(size_bytes).await;
|
||||
|
||||
let entry = CachedDataset {
|
||||
name: name.to_string(),
|
||||
schema,
|
||||
batches,
|
||||
row_count,
|
||||
size_bytes,
|
||||
last_accessed: std::time::Instant::now(),
|
||||
};
|
||||
|
||||
tracing::info!("cached '{}': {} rows, {:.1} MB",
|
||||
name, row_count, size_bytes as f64 / 1024.0 / 1024.0);
|
||||
|
||||
let mut cache = self.datasets.write().await;
|
||||
cache.insert(name.to_string(), entry);
|
||||
}
|
||||
|
||||
/// Get a cached dataset, updating access time.
|
||||
pub async fn get(&self, name: &str) -> Option<CachedDataset> {
|
||||
let mut cache = self.datasets.write().await;
|
||||
if let Some(entry) = cache.get_mut(name) {
|
||||
entry.last_accessed = std::time::Instant::now();
|
||||
Some(entry.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a dataset is cached.
|
||||
pub async fn contains(&self, name: &str) -> bool {
|
||||
self.datasets.read().await.contains_key(name)
|
||||
}
|
||||
|
||||
/// Evict a specific dataset.
|
||||
pub async fn evict(&self, name: &str) -> bool {
|
||||
let mut cache = self.datasets.write().await;
|
||||
cache.remove(name).is_some()
|
||||
}
|
||||
|
||||
/// Get cache stats.
|
||||
pub async fn stats(&self) -> CacheStats {
|
||||
let cache = self.datasets.read().await;
|
||||
let total_bytes: usize = cache.values().map(|d| d.size_bytes).sum();
|
||||
let total_rows: usize = cache.values().map(|d| d.row_count).sum();
|
||||
CacheStats {
|
||||
datasets: cache.len(),
|
||||
total_bytes,
|
||||
total_rows,
|
||||
max_bytes: self.max_bytes,
|
||||
names: cache.keys().cloned().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Evict least-recently-used entries to make room for `needed_bytes`.
|
||||
async fn evict_to_fit(&self, needed_bytes: usize) {
|
||||
let mut cache = self.datasets.write().await;
|
||||
let current: usize = cache.values().map(|d| d.size_bytes).sum();
|
||||
|
||||
if current + needed_bytes <= self.max_bytes {
|
||||
return;
|
||||
}
|
||||
|
||||
// Sort by last accessed (oldest first)
|
||||
let mut entries: Vec<(String, std::time::Instant, usize)> = cache.iter()
|
||||
.map(|(k, v)| (k.clone(), v.last_accessed, v.size_bytes))
|
||||
.collect();
|
||||
entries.sort_by_key(|(_, t, _)| *t);
|
||||
|
||||
let mut freed = 0usize;
|
||||
let target = (current + needed_bytes).saturating_sub(self.max_bytes);
|
||||
|
||||
for (name, _, size) in entries {
|
||||
if freed >= target {
|
||||
break;
|
||||
}
|
||||
tracing::info!("evicting '{}' from cache ({:.1} MB)", name, size as f64 / 1024.0 / 1024.0);
|
||||
cache.remove(&name);
|
||||
freed += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct CacheStats {
|
||||
pub datasets: usize,
|
||||
pub total_bytes: usize,
|
||||
pub total_rows: usize,
|
||||
pub max_bytes: usize,
|
||||
pub names: Vec<String>,
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
use catalogd::registry::Registry;
|
||||
use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl};
|
||||
use datafusion::datasource::MemTable;
|
||||
use datafusion::datasource::file_format::parquet::ParquetFormat;
|
||||
use datafusion::prelude::*;
|
||||
use object_store::ObjectStore;
|
||||
use std::sync::Arc;
|
||||
use storaged::registry::BucketRegistry;
|
||||
use url::Url;
|
||||
|
||||
use crate::cache::MemCache;
|
||||
use crate::delta;
|
||||
|
||||
const STORE_SCHEME: &str = "lakehouse";
|
||||
|
||||
/// URL scheme used to register a bucket's object store with DataFusion.
|
||||
/// Bucket names safely embed in URL host components after sanitizing
|
||||
/// `:` (used in `profile:user`) to `-`.
|
||||
fn bucket_scheme(bucket: &str) -> String {
|
||||
format!("{STORE_SCHEME}-{}", bucket.replace(':', "-"))
|
||||
}
|
||||
|
||||
/// Build a `ListingTableUrl`-shaped string for an object in a bucket.
|
||||
fn bucket_object_url(bucket: &str, key: &str) -> String {
|
||||
format!("{}://data/{}", bucket_scheme(bucket), key)
|
||||
}
|
||||
|
||||
/// Build the registration base URL for a bucket.
|
||||
fn bucket_base_url(bucket: &str) -> String {
|
||||
format!("{}://data/", bucket_scheme(bucket))
|
||||
}
|
||||
|
||||
/// Query engine with in-memory cache and delta merge support.
|
||||
/// Federation layer 2: holds a `BucketRegistry` so cross-bucket queries
|
||||
/// route reads to the right ObjectStore based on `ObjectRef.bucket`.
|
||||
#[derive(Clone)]
|
||||
pub struct QueryEngine {
|
||||
registry: Registry,
|
||||
buckets: Arc<BucketRegistry>,
|
||||
/// Primary bucket store cached for backward-compat callers (compact,
|
||||
/// workspace ops) that haven't been migrated to bucket-aware paths.
|
||||
store: Arc<dyn ObjectStore>,
|
||||
cache: MemCache,
|
||||
}
|
||||
|
||||
impl QueryEngine {
|
||||
pub fn new(registry: Registry, buckets: Arc<BucketRegistry>, cache: MemCache) -> Self {
|
||||
let store = buckets.default_store();
|
||||
Self { registry, buckets, store, cache }
|
||||
}
|
||||
|
||||
pub fn cache(&self) -> &MemCache {
|
||||
&self.cache
|
||||
}
|
||||
|
||||
/// Backward-compat: returns the primary bucket's store. Bucket-aware
|
||||
/// callers should use `engine.buckets()` directly.
|
||||
pub fn store(&self) -> &Arc<dyn ObjectStore> {
|
||||
&self.store
|
||||
}
|
||||
|
||||
pub fn buckets(&self) -> &Arc<BucketRegistry> {
|
||||
&self.buckets
|
||||
}
|
||||
|
||||
/// Catalog registry — used by compact/tombstone integration.
|
||||
pub fn catalog(&self) -> &Registry {
|
||||
&self.registry
|
||||
}
|
||||
|
||||
/// Execute a SQL query. Uses cache for hot data, falls back to Parquet.
|
||||
pub async fn query(&self, sql: &str) -> Result<Vec<arrow::array::RecordBatch>, String> {
|
||||
let ctx = self.build_context().await?;
|
||||
let df = ctx.sql(sql).await.map_err(|e| format!("SQL error: {e}"))?;
|
||||
let batches = df.collect().await.map_err(|e| format!("execution error: {e}"))?;
|
||||
Ok(batches)
|
||||
}
|
||||
|
||||
/// Pin a dataset into the memory cache.
|
||||
pub async fn pin_dataset(&self, name: &str) -> Result<(), String> {
|
||||
let ctx = SessionContext::new();
|
||||
|
||||
// Register every bucket so a multi-bucket dataset can be pinned.
|
||||
for info in self.buckets.list().await {
|
||||
let url = Url::parse(&bucket_base_url(&info.name))
|
||||
.map_err(|e| format!("invalid store url: {e}"))?;
|
||||
let store = self.buckets.get(&info.name)
|
||||
.map_err(|e| format!("registry inconsistency: {e}"))?;
|
||||
ctx.runtime_env().register_object_store(&url, store);
|
||||
}
|
||||
|
||||
let dataset = self.registry.get_by_name(name).await
|
||||
.ok_or_else(|| format!("dataset not found: {name}"))?;
|
||||
|
||||
if dataset.objects.is_empty() {
|
||||
return Err(format!("dataset '{name}' has no objects"));
|
||||
}
|
||||
|
||||
let opts = ListingOptions::new(Arc::new(ParquetFormat::default()));
|
||||
let table_paths: Vec<ListingTableUrl> = dataset.objects.iter()
|
||||
.filter_map(|o| {
|
||||
let bucket = if o.bucket.is_empty() { "primary" } else { &o.bucket };
|
||||
ListingTableUrl::parse(&bucket_object_url(bucket, &o.key)).ok()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let schema = opts.infer_schema(&ctx.state(), &table_paths[0]).await
|
||||
.map_err(|e| format!("schema inference: {e}"))?;
|
||||
let config = ListingTableConfig::new_with_multi_paths(table_paths)
|
||||
.with_listing_options(opts)
|
||||
.with_schema(schema.clone());
|
||||
let table = ListingTable::try_new(config).map_err(|e| format!("table: {e}"))?;
|
||||
|
||||
ctx.register_table(name, Arc::new(table)).map_err(|e| format!("register: {e}"))?;
|
||||
let df = ctx.sql(&format!("SELECT * FROM {name}")).await.map_err(|e| format!("read: {e}"))?;
|
||||
let batches = df.collect().await.map_err(|e| format!("collect: {e}"))?;
|
||||
|
||||
self.cache.put(name, schema, batches).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_context(&self) -> Result<SessionContext, String> {
|
||||
let ctx = SessionContext::new();
|
||||
|
||||
// Phase E: snapshot tombstones by dataset before registering tables
|
||||
// so we can wrap tombstoned tables in a filter view. The underlying
|
||||
// base table is registered under an internal name `__raw__{dataset}`
|
||||
// and the public `{dataset}` name becomes the filtered view.
|
||||
let all_dataset_names: Vec<String> = self.registry.list().await
|
||||
.iter().map(|d| d.name.clone()).collect();
|
||||
let tombstones_by_dataset = self
|
||||
.registry
|
||||
.tombstones()
|
||||
.all_grouped(&all_dataset_names)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
// Federation layer 2: register every configured bucket as its own
|
||||
// DataFusion ObjectStore under a distinct URL scheme. Each
|
||||
// dataset's ObjectRef.bucket determines which store DataFusion
|
||||
// routes to at scan time.
|
||||
let bucket_infos = self.buckets.list().await;
|
||||
for info in &bucket_infos {
|
||||
let url_str = bucket_base_url(&info.name);
|
||||
let url = Url::parse(&url_str)
|
||||
.map_err(|e| format!("invalid store url '{url_str}': {e}"))?;
|
||||
// unknown-bucket here would be a config invariant violation,
|
||||
// since we just listed them — propagate as an error to surface it.
|
||||
let store = self.buckets.get(&info.name)
|
||||
.map_err(|e| format!("registry inconsistency: {e}"))?;
|
||||
ctx.runtime_env().register_object_store(&url, store);
|
||||
}
|
||||
|
||||
// Backward-compat: also register the legacy `lakehouse://data/`
|
||||
// URL pointing at primary, in case any code path still constructs
|
||||
// ListingTableUrls without a bucket prefix.
|
||||
let legacy_url = Url::parse(&format!("{STORE_SCHEME}://data/"))
|
||||
.map_err(|e| format!("invalid legacy store url: {e}"))?;
|
||||
ctx.runtime_env().register_object_store(&legacy_url, self.store.clone());
|
||||
|
||||
let datasets = self.registry.list().await;
|
||||
for dataset in &datasets {
|
||||
if dataset.objects.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check cache first — cached batches are bucket-agnostic since
|
||||
// they're already materialized in memory.
|
||||
if let Some(cached) = self.cache.get(&dataset.name).await {
|
||||
let delta_batches = delta::load_deltas(&self.store, &dataset.name).await.unwrap_or_default();
|
||||
|
||||
let mut all_batches = cached.batches;
|
||||
all_batches.extend(delta_batches);
|
||||
|
||||
let mem_table = MemTable::try_new(cached.schema, vec![all_batches])
|
||||
.map_err(|e| format!("MemTable error for {}: {e}", dataset.name))?;
|
||||
ctx.register_table(&dataset.name, Arc::new(mem_table))
|
||||
.map_err(|e| format!("register cached {}: {e}", dataset.name))?;
|
||||
|
||||
tracing::debug!("using cached table: {}", dataset.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Build ListingTable URLs that include the per-object bucket
|
||||
// scheme. DataFusion routes each scan to the right store
|
||||
// automatically based on the URL.
|
||||
let opts = ListingOptions::new(Arc::new(ParquetFormat::default()));
|
||||
let table_paths: Vec<ListingTableUrl> = dataset.objects.iter()
|
||||
.filter_map(|o| {
|
||||
let bucket = if o.bucket.is_empty() { "primary" } else { &o.bucket };
|
||||
ListingTableUrl::parse(&bucket_object_url(bucket, &o.key)).ok()
|
||||
})
|
||||
.collect();
|
||||
|
||||
if table_paths.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let schema = opts.infer_schema(&ctx.state(), &table_paths[0]).await
|
||||
.map_err(|e| format!("schema inference failed for {}: {e}", dataset.name))?;
|
||||
let config = ListingTableConfig::new_with_multi_paths(table_paths)
|
||||
.with_listing_options(opts)
|
||||
.with_schema(schema);
|
||||
let table = ListingTable::try_new(config)
|
||||
.map_err(|e| format!("table creation failed for {}: {e}", dataset.name))?;
|
||||
|
||||
// Decide the registration name: if this dataset has any
|
||||
// tombstones, the raw table gets an internal name and the
|
||||
// public name becomes a filtered view.
|
||||
let tombstone_entry = tombstones_by_dataset.get(&dataset.name);
|
||||
let register_name = if tombstone_entry.is_some() {
|
||||
format!("__raw__{}", dataset.name)
|
||||
} else {
|
||||
dataset.name.clone()
|
||||
};
|
||||
|
||||
if let Err(e) = ctx.register_table(register_name.as_str(), Arc::new(table)) {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("already exists") {
|
||||
tracing::debug!("skip duplicate manifest registration: {}", register_name);
|
||||
continue;
|
||||
} else {
|
||||
return Err(format!("table registration failed for {}: {}", register_name, msg));
|
||||
}
|
||||
}
|
||||
|
||||
// If there are tombstones, register the public name as a
|
||||
// filtered view that excludes tombstoned row_key_values.
|
||||
if let Some((key_col, values)) = tombstone_entry {
|
||||
// Build WHERE NOT IN (...) — quote values to be SQL-safe.
|
||||
// For string keys this is a literal list; for integer keys
|
||||
// CAST(col AS VARCHAR) makes the comparison unambiguous.
|
||||
let quoted: Vec<String> = values.iter()
|
||||
.map(|v| format!("'{}'", v.replace('\'', "''")))
|
||||
.collect();
|
||||
let sql = format!(
|
||||
"SELECT * FROM \"{}\" WHERE CAST(\"{}\" AS VARCHAR) NOT IN ({})",
|
||||
register_name, key_col, quoted.join(", "),
|
||||
);
|
||||
tracing::debug!(
|
||||
"tombstone filter for '{}': {} row_keys excluded",
|
||||
dataset.name, values.len(),
|
||||
);
|
||||
match ctx.sql(&sql).await {
|
||||
Ok(df) => {
|
||||
if let Err(e) = ctx.register_table(dataset.name.as_str(), df.into_view()) {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("already exists") {
|
||||
tracing::debug!("skip duplicate tombstone view: {}", dataset.name);
|
||||
} else {
|
||||
tracing::warn!("tombstone view registration failed for {}: {msg}", dataset.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("tombstone view SQL failed for '{}': {e}", dataset.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase D: register every AiView as a DataFusion view that wraps
|
||||
// the base table with the safe projection + filter + redactions.
|
||||
for view in self.registry.list_views().await {
|
||||
// Build the SELECT clause with column whitelist and any
|
||||
// redaction expressions. Each column is either the raw column
|
||||
// name or an expression aliased back to the column name so
|
||||
// downstream queries see the same shape.
|
||||
let select_cols: Vec<String> = view.columns.iter().map(|col| {
|
||||
match view.column_redactions.get(col) {
|
||||
None => format!("\"{}\"", col),
|
||||
Some(shared::types::Redaction::Null) => {
|
||||
format!("CAST(NULL AS VARCHAR) AS \"{}\"", col)
|
||||
}
|
||||
Some(shared::types::Redaction::Hash) => {
|
||||
// DataFusion has digest('sha256', value) — but column
|
||||
// type might not be string. Cast first to be safe.
|
||||
format!("digest(CAST(\"{}\" AS VARCHAR), 'sha256') AS \"{}\"", col, col)
|
||||
}
|
||||
Some(shared::types::Redaction::Mask { keep_prefix, keep_suffix }) => {
|
||||
// Keep first N + last M chars, replace middle with stars.
|
||||
// CASE NULL through with COALESCE for null-safe behavior.
|
||||
format!(
|
||||
"CASE WHEN \"{c}\" IS NULL THEN NULL ELSE \
|
||||
concat(\
|
||||
substr(CAST(\"{c}\" AS VARCHAR), 1, {p}), \
|
||||
repeat('*', greatest(0, length(CAST(\"{c}\" AS VARCHAR)) - {p} - {s})), \
|
||||
substr(CAST(\"{c}\" AS VARCHAR), greatest(1, length(CAST(\"{c}\" AS VARCHAR)) - {s} + 1))\
|
||||
) END AS \"{c}\"",
|
||||
c = col, p = keep_prefix, s = keep_suffix,
|
||||
)
|
||||
}
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let where_clause = view.row_filter
|
||||
.as_deref()
|
||||
.map(|f| format!(" WHERE {}", f))
|
||||
.unwrap_or_default();
|
||||
let view_sql = format!(
|
||||
"SELECT {} FROM \"{}\"{}",
|
||||
select_cols.join(", "),
|
||||
view.base_dataset,
|
||||
where_clause,
|
||||
);
|
||||
|
||||
tracing::debug!("registering AiView '{}': {}", view.name, view_sql);
|
||||
|
||||
let df = match ctx.sql(&view_sql).await {
|
||||
Ok(df) => df,
|
||||
Err(e) => {
|
||||
tracing::warn!("view '{}' SQL invalid, skipping: {e}", view.name);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Err(e) = ctx.register_table(view.name.as_str(), df.into_view()) {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("already exists") {
|
||||
tracing::debug!("skip duplicate view registration: {}", view.name);
|
||||
} else {
|
||||
tracing::warn!("view '{}' registration failed: {e}", view.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ctx)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user