observer: add /relevance heuristic filter for adjacency pollution
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
Matrix retrieval often surfaces high-cosine chunks that are about
symbols the focus file IMPORTS but doesn't define. The reviewer LLM
then hallucinates those imported-crate internals as in-file content
("I see main.rs does X" when X lives in queryd::context).
mcp-server/relevance.ts — pure scorer with five signals:
path_match +1.0 chunk source/doc_id encodes focus path
defined_match +0.6 chunk text mentions focus.defined_symbols
token_overlap +0.4 jaccard of non-stopword tokens
prefix_match +0.3 shared first-2-segment prefix
import_only -0.5 mentions only imported symbols (pollution)
Default threshold 0.3 — tuned empirically on the gateway/main.rs case.
Also fixes a regex bug in the import extractor: the character class
was lowercase-only, so `use catalogd::Registry;` silently never
matched (regex backed off when it hit the uppercase R). Caught by
the test suite.
observer.ts — POST /relevance endpoint wraps filterChunks().
scrum_master_pipeline.ts — fetchMatrixContext gains optional
focusContent param; calls /relevance after collecting allHits and
before sort+top. Opt-out via LH_RELEVANCE_FILTER=0; threshold via
LH_RELEVANCE_THRESHOLD. Fall-open on observer failure.
9 unit tests, all green. Live probe on real shape correctly drops
a 0.7-cosine adjacency-pollution chunk while keeping in-focus hits.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -499,6 +499,7 @@ async function fetchMatrixContext(
|
||||
filePath: string,
|
||||
topPerCorpus = 3,
|
||||
topOverall = 8,
|
||||
focusContent?: string,
|
||||
): Promise<MatrixContext> {
|
||||
const t0 = Date.now();
|
||||
const corpora = MATRIX_CORPORA_FOR_TASK[taskClass] ?? [];
|
||||
@@ -548,9 +549,48 @@ async function fetchMatrixContext(
|
||||
errors.pathway_memory = e.message;
|
||||
}
|
||||
|
||||
// Optional adjacency-pollution filter via observer /relevance.
|
||||
// Drops chunks that scored well on cosine but are about symbols the
|
||||
// focus file only IMPORTS (not defines). Opt-out with
|
||||
// LH_RELEVANCE_FILTER=0 for A/B comparison.
|
||||
const relevanceEnabled = process.env.LH_RELEVANCE_FILTER !== "0";
|
||||
let filteredHits = allHits;
|
||||
let droppedCount = 0;
|
||||
if (relevanceEnabled && allHits.length > 0) {
|
||||
try {
|
||||
const r = await fetch(`${OBSERVER_URL}/relevance`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
focus_file: { path: filePath, content: focusContent ?? "" },
|
||||
chunks: allHits.map(h => ({
|
||||
source: h.source_corpus,
|
||||
doc_id: h.doc_id,
|
||||
text: h.text,
|
||||
score: h.score,
|
||||
})),
|
||||
threshold: Number(process.env.LH_RELEVANCE_THRESHOLD ?? 0.3),
|
||||
}),
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
if (r.ok) {
|
||||
const fr: any = await r.json();
|
||||
const keptKeys = new Set((fr.kept ?? []).map((c: any) => `${c.source}|${c.doc_id}`));
|
||||
filteredHits = allHits.filter(h => keptKeys.has(`${h.source_corpus}|${h.doc_id}`));
|
||||
droppedCount = (fr.dropped ?? []).length;
|
||||
} else {
|
||||
errors.relevance_filter = `HTTP ${r.status}`;
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Fall-open: filter failure must not block the pipeline.
|
||||
errors.relevance_filter = e.message;
|
||||
}
|
||||
}
|
||||
|
||||
// Sort all hits by score desc, take top N
|
||||
allHits.sort((a, b) => b.score - a.score);
|
||||
const topHits = allHits.slice(0, topOverall);
|
||||
filteredHits.sort((a, b) => b.score - a.score);
|
||||
const topHits = filteredHits.slice(0, topOverall);
|
||||
if (droppedCount > 0) byCorpus._relevance_dropped = droppedCount;
|
||||
|
||||
return {
|
||||
hits: topHits,
|
||||
@@ -1349,7 +1389,7 @@ Respond with markdown. Be specific, not generic. Cite file-region + PRD-chunk-of
|
||||
// source so retrieval anchors against both the metadata and the
|
||||
// actual code being reviewed.
|
||||
const matrixQuery = `${taskClass} ${rel} ${content.slice(0, 500)}`;
|
||||
const matrixCtx = await fetchMatrixContext(matrixQuery, taskClass, rel, 3, 8);
|
||||
const matrixCtx = await fetchMatrixContext(matrixQuery, taskClass, rel, 3, 8, content);
|
||||
provenApproachesPreamble = buildMatrixPreamble(matrixCtx);
|
||||
const corporaSummary = Object.entries(matrixCtx.by_corpus)
|
||||
.map(([k, v]) => `${k.split("_v")[0]}=${v}`).join(" ");
|
||||
|
||||
Reference in New Issue
Block a user