profiler: multi-source quote aggregation with confidence scoring
Replaces the single-source quote feed with a 3-source consensus system,
the differentiator for the index: pull each ticker from independent
feeds, take the median, and score confidence by how many corroborate it.
Disagreement becomes a signal instead of a silently-wrong number.
Context: Stooq retired its free q/l/ CSV quote API (404s server-side;
q/d/l behind a JS challenge), which broke the profiler basket. The
interim Yahoo-only fix worked but had a hidden wrong-baseline bug —
Yahoo's chartPreviousClose over a 5d window references the pre-window
close, so day-change read +11.65% for CAT when the real figure is
+1.11%. Cross-checking against Nasdaq + CNBC exposes exactly that class
of quiet error, which a single source can never self-report.
- quotes.ts — Yahoo + Nasdaq + CNBC fetchers (all keyless, browser UA;
Yahoo prev-close now derived from the daily candle, not the buggy
meta field), median consensus, confidence = corroborating sources / 3,
outlier flagging, and a bun:sqlite hourly time-series cache.
- quote_ingest.ts + ops/systemd/lakehouse-quote-ingest.{service,timer} —
hourly snapshot into the cache (real-time not required per product
call). Runs as root to match the mcp-server that co-owns the db;
busy_timeout serializes the two writers.
- /intelligence/ticker_quotes — serves cache-first (fast), aggregates
cold tickers on-demand once and caches them. New /quote_ingest admin
trigger.
- profiler.html — per-card confidence badge (green/amber/red) + a
per-source breakdown tooltip; "3-source consensus" labeling.
- entity.ts — the earlier Stooq->Yahoo swap for the single-ticker brief
(kept fetchStooqQuote dead-but-reversible).
Verified live on devop.live/lakehouse/profiler: liquid names read
100% / 3-source consensus; thin microcaps (BLPG, ESHSF — Yahoo-only)
honestly read 33% / 1-source. Full basket (17 tickers) seeded; hourly
timer armed. Cache db (data/_quote_cache/) is gitignored.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+57
-4
@@ -1140,6 +1140,56 @@ export async function fetchStooqQuote(ticker: string): Promise<{
|
||||
}
|
||||
}
|
||||
|
||||
// Yahoo Finance v8 chart API — the replacement quote source (2026-06-18).
|
||||
// Stooq retired its free `q/l/` light-quote CSV endpoint (now 404s for
|
||||
// non-browser clients; the historical `q/d/l/` CSV sits behind a JS
|
||||
// browser-verification challenge), so fetchStooqQuote returns null for
|
||||
// every symbol and the profiler basket renders no quotes. The v8 chart
|
||||
// endpoint is free, key-less, single-symbol JSON and exposes everything
|
||||
// the basket needs in `.chart.result[0].meta`: regularMarketPrice (last
|
||||
// price), chartPreviousClose (for the conventional day-change), plus
|
||||
// day high/low/volume. Same single-symbol shape as Stooq, so the
|
||||
// per-ticker fan-out in /intelligence/ticker_quotes is unchanged.
|
||||
//
|
||||
// fetchStooqQuote is kept above (dead upstream, not deleted) so the
|
||||
// swap is reversible if a keyed provider replaces this later.
|
||||
export async function fetchYahooQuote(ticker: string): Promise<{
|
||||
price?: number;
|
||||
prev_close?: number;
|
||||
price_date?: string;
|
||||
open?: number;
|
||||
high?: number;
|
||||
low?: number;
|
||||
volume?: number;
|
||||
} | null> {
|
||||
try {
|
||||
const res = await fetch(
|
||||
`https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(ticker.toUpperCase())}?range=5d&interval=1d`,
|
||||
{ headers: { "User-Agent": UA }, signal: AbortSignal.timeout(8000) },
|
||||
);
|
||||
if (!res.ok) return null;
|
||||
const j: any = await res.json();
|
||||
const r = j?.chart?.result?.[0];
|
||||
const m = r?.meta;
|
||||
if (!m || typeof m.regularMarketPrice !== "number") return null;
|
||||
// Last daily candle for the open (meta has no regularMarketOpen).
|
||||
const q = r?.indicators?.quote?.[0] ?? {};
|
||||
const lastOpen = Array.isArray(q.open) ? q.open[q.open.length - 1] : undefined;
|
||||
const t = typeof m.regularMarketTime === "number" ? m.regularMarketTime : null;
|
||||
return {
|
||||
price: m.regularMarketPrice,
|
||||
prev_close: typeof m.chartPreviousClose === "number" ? m.chartPreviousClose : undefined,
|
||||
price_date: t ? new Date(t * 1000).toISOString().slice(0, 10) : undefined,
|
||||
open: typeof lastOpen === "number" ? lastOpen : undefined,
|
||||
high: typeof m.regularMarketDayHigh === "number" ? m.regularMarketDayHigh : undefined,
|
||||
low: typeof m.regularMarketDayLow === "number" ? m.regularMarketDayLow : undefined,
|
||||
volume: typeof m.regularMarketVolume === "number" ? m.regularMarketVolume : undefined,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Cheap name-to-ticker lookup for batch use (e.g. profiler index over
|
||||
// 200 contractors). Hits the in-memory SEC tickers index and the
|
||||
// KNOWN_PARENT_MAP. No SEC profile fetch, no Stooq fetch. Returns
|
||||
@@ -1198,14 +1248,17 @@ export async function fetchTickerBrief(name: string): Promise<TickerBrief> {
|
||||
}
|
||||
const [profile, quote] = await Promise.all([
|
||||
fetchSecProfile(hit.cik_str),
|
||||
fetchStooqQuote(hit.ticker),
|
||||
fetchYahooQuote(hit.ticker), // Stooq CSV retired 2026-06-18; see fetchYahooQuote
|
||||
]);
|
||||
const cap_proxy =
|
||||
quote?.price && quote?.volume ? quote.price * quote.volume : undefined;
|
||||
// Day change vs previous close (conventional); fall back to vs-open.
|
||||
const day_change_pct =
|
||||
quote?.price && quote?.open && quote.open > 0
|
||||
? ((quote.price - quote.open) / quote.open) * 100
|
||||
: undefined;
|
||||
quote?.price && quote?.prev_close && quote.prev_close > 0
|
||||
? ((quote.price - quote.prev_close) / quote.prev_close) * 100
|
||||
: quote?.price && quote?.open && quote.open > 0
|
||||
? ((quote.price - quote.open) / quote.open) * 100
|
||||
: undefined;
|
||||
const brief: TickerBrief = {
|
||||
source: "sec+stooq",
|
||||
fetched_at: now,
|
||||
|
||||
Reference in New Issue
Block a user