scripts: chicago analyzer field-name fixes + vectorize sanitizer hardening
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts

Two small fixes surfaced during smoke testing:

analyze_chicago_contracts.ts: permit field is contact_1_name not
contact_1; reported_cost is integer-string. Fixed filter (was rejecting
all 2853 permits) and contractor extraction (was empty).

vectorize_raw_corpus.ts: sanitize() expanded to strip control chars +
ALL backslashes (kills incomplete \uXXXX escapes) + UTF-16 surrogates
(unpaired surrogates from emoji split by truncate boundary). Llm_team
response cache had docs with all three pollution shapes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-25 19:34:45 -05:00
co-authored by Claude Opus 4.7
parent 6ac7f61819
commit 779158a09b
2 changed files with 21 additions and 8 deletions
+15 -3
View File
@@ -46,6 +46,18 @@ function truncate(s: string, n = 4000): string {
return s == null ? "" : (s.length > n ? s.slice(0, n) : s);
}
// Sanitize text before posting as JSON. Strips control chars and
// drops incomplete \uXXXX escape sequences which break Rust's
// serde JSON parser at the gateway. Llm_team response cache had
// rows with truncated \u escapes that 400'd the whole batch.
function sanitize(s: string): string {
if (!s) return "";
return s
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "") // strip control chars
.replace(/\\/g, "") // strip ALL backslashes — kills any malformed \uXXXX in source data
.replace(/[\uD800-\uDFFF]/g, ""); // strip UTF-16 surrogates (lone ones from emoji split by truncate)
}
// ─── EXTRACTORS — one per source ───
// Each shapes raw rows into {id, text} for the gateway's chunker.
@@ -129,13 +141,13 @@ function extractLlmTeamResponseCache(raw: string): Doc[] {
const text = [
`Cached response ${r.cache_key ?? r.id ?? i}`,
`Created: ${r.created_at ?? "?"}`,
r.prompt ? `Prompt: ${truncate(r.prompt, 1500)}` : "",
r.response ? `Response: ${truncate(r.response, 2500)}` : "",
r.prompt ? `Prompt: ${sanitize(truncate(r.prompt, 1500))}` : "",
r.response ? `Response: ${sanitize(truncate(r.response, 2500))}` : "",
r.model ? `Model: ${r.model}` : "",
].filter(Boolean).join("\n");
return { id: `resp_${r.cache_key ?? r.id ?? i}`, text };
} catch {
return { id: `resp_${i}`, text: line.slice(0, 2000) };
return { id: `resp_${i}`, text: sanitize(line.slice(0, 2000)) };
}
});
}