scripts: chicago analyzer field-name fixes + vectorize sanitizer hardening
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
Two small fixes surfaced during smoke testing: analyze_chicago_contracts.ts: permit field is contact_1_name not contact_1; reported_cost is integer-string. Fixed filter (was rejecting all 2853 permits) and contractor extraction (was empty). vectorize_raw_corpus.ts: sanitize() expanded to strip control chars + ALL backslashes (kills incomplete \uXXXX escapes) + UTF-16 surrogates (unpaired surrogates from emoji split by truncate boundary). Llm_team response cache had docs with all three pollution shapes. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -46,6 +46,18 @@ function truncate(s: string, n = 4000): string {
|
||||
return s == null ? "" : (s.length > n ? s.slice(0, n) : s);
|
||||
}
|
||||
|
||||
// Sanitize text before posting as JSON. Strips control chars and
|
||||
// drops incomplete \uXXXX escape sequences which break Rust's
|
||||
// serde JSON parser at the gateway. Llm_team response cache had
|
||||
// rows with truncated \u escapes that 400'd the whole batch.
|
||||
function sanitize(s: string): string {
|
||||
if (!s) return "";
|
||||
return s
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "") // strip control chars
|
||||
.replace(/\\/g, "") // strip ALL backslashes — kills any malformed \uXXXX in source data
|
||||
.replace(/[\uD800-\uDFFF]/g, ""); // strip UTF-16 surrogates (lone ones from emoji split by truncate)
|
||||
}
|
||||
|
||||
// ─── EXTRACTORS — one per source ───
|
||||
// Each shapes raw rows into {id, text} for the gateway's chunker.
|
||||
|
||||
@@ -129,13 +141,13 @@ function extractLlmTeamResponseCache(raw: string): Doc[] {
|
||||
const text = [
|
||||
`Cached response ${r.cache_key ?? r.id ?? i}`,
|
||||
`Created: ${r.created_at ?? "?"}`,
|
||||
r.prompt ? `Prompt: ${truncate(r.prompt, 1500)}` : "",
|
||||
r.response ? `Response: ${truncate(r.response, 2500)}` : "",
|
||||
r.prompt ? `Prompt: ${sanitize(truncate(r.prompt, 1500))}` : "",
|
||||
r.response ? `Response: ${sanitize(truncate(r.response, 2500))}` : "",
|
||||
r.model ? `Model: ${r.model}` : "",
|
||||
].filter(Boolean).join("\n");
|
||||
return { id: `resp_${r.cache_key ?? r.id ?? i}`, text };
|
||||
} catch {
|
||||
return { id: `resp_${i}`, text: line.slice(0, 2000) };
|
||||
return { id: `resp_${i}`, text: sanitize(line.slice(0, 2000)) };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user