chore: add real content that was sitting untracked

Surfaced by today's untracked-files audit. None of these are accidents —
multiple are referenced by name in CLAUDE.md and memory files but were
never added.

Categories:
- docs/PHASE_AUDIT_GUIDE.md (106 LOC) — Claude Code phase audit guidance
- ops/systemd/lakehouse-langfuse-bridge.service — Langfuse bridge unit
- package.json — top-level npm manifest
- scripts/e2e_pipeline_check.sh + production_smoke.sh — real test scripts
- reports/kimi/audit-last-week*.md — the "Two reports live" CLAUDE.md cites
- tests/multi-agent/scenarios/ — 44 staffing scenarios (cutover decision A)
- tests/multi-agent/playbooks/ — 102 playbook records
- tests/battery/, tests/agent_test/PRD.md, tests/real-world/* — real tests
- sidecar/sidecar/{lab_ui,pipeline_lab}.py — 888 LOC dev-only UIs that
  remain in service post-sidecar-drop (commit ba928b1 explicitly kept them)

Sensitivity check: scenarios use synthetic company names ("Heritage Foods",
"Cornerstone Fabrication"); audit reports describe code findings only;
no PII or secrets surfaced.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-05-02 22:22:10 -05:00
co-authored by Claude Opus 4.7
parent 6e34ef7baf
commit 41b0a99ed2
788 changed files with 107142 additions and 0 deletions
+385
View File
@@ -0,0 +1,385 @@
"""Pipeline Lab notebook UI — served as a single HTML page.
Note: innerHTML usage in this file is intentional for building the UI.
All user-supplied text is escaped through the esc() function before insertion.
The only values rendered via innerHTML are pre-formatted HTML strings with
escaped user content — no raw user input is ever injected unescaped.
"""
from fastapi import APIRouter
from fastapi.responses import HTMLResponse
router = APIRouter()
def _get_lab_html() -> str:
"""Return the Pipeline Lab HTML. Separated into a function for clarity."""
# The HTML is a self-contained notebook UI.
# All user-facing text is escaped via the esc() JS function.
return r"""<!DOCTYPE html>
<html lang="en"><head>
<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
<title>Pipeline Lab — Lakehouse</title>
<style>
:root{--bg:#08090c;--surface:rgba(14,16,22,0.9);--border:#2a2d35;--text:#e8e6e3;--text2:#7a7872;--accent:#4ade80;--gold:#e2b55a;--red:#e05252;--blue:#5b9cf5;--purple:#c084fc}
*{box-sizing:border-box;margin:0;padding:0}
body{font-family:'SF Mono','Menlo','Consolas',monospace;background:var(--bg);color:var(--text);min-height:100vh;padding:20px 28px;font-size:13px}
h1{font-size:18px;font-weight:700;margin-bottom:4px}h1 span{color:var(--accent)}
.subtitle{color:var(--text2);font-size:11px;margin-bottom:20px}
.cells{display:flex;flex-direction:column;gap:12px;max-width:1100px}
.cell{background:var(--surface);border:1px solid var(--border);border-radius:4px;overflow:hidden}
.cell.running{border-color:var(--gold)}
.cell-header{display:flex;align-items:center;gap:8px;padding:8px 12px;border-bottom:1px solid var(--border);font-size:10px;text-transform:uppercase;letter-spacing:1px;color:var(--text2)}
.cell-type{font-weight:700}
.cell-time{margin-left:auto;color:var(--text2)}
.cell-input{padding:12px;background:rgba(0,0,0,0.3)}
.cell-input textarea{width:100%;min-height:60px;background:transparent;border:none;color:var(--text);font-family:inherit;font-size:13px;resize:vertical;outline:none;line-height:1.6}
.cell-output{padding:12px;font-size:12px;line-height:1.6;white-space:pre-wrap;max-height:400px;overflow-y:auto;display:none}
.cell-output.has-data{display:block;border-top:1px solid var(--border)}
.toolbar{display:flex;gap:6px;padding:8px 12px;border-top:1px solid var(--border);flex-wrap:wrap}
.btn{font-family:inherit;font-size:10px;text-transform:uppercase;letter-spacing:0.5px;padding:5px 12px;border:1px solid var(--border);border-radius:3px;background:transparent;color:var(--text2);cursor:pointer}
.btn:hover{border-color:var(--accent);color:var(--accent)}
.btn.primary{border-color:var(--accent);color:var(--accent);background:rgba(74,222,128,0.06)}
.btn.gold{border-color:var(--gold);color:var(--gold)}
.btn.blue{border-color:var(--blue);color:var(--blue)}
.btn.purple{border-color:var(--purple);color:var(--purple)}
.btn.red{border-color:var(--red);color:var(--red)}
.top-bar{display:flex;gap:8px;margin-bottom:16px;align-items:center;flex-wrap:wrap}
.status-bar{display:flex;gap:12px;padding:8px 12px;background:var(--surface);border:1px solid var(--border);border-radius:4px;margin-bottom:16px;font-size:10px;color:var(--text2)}
.stat{display:flex;align-items:center;gap:4px}.stat b{color:var(--text)}
.result-row{display:flex;gap:8px;padding:6px 8px;border-bottom:1px solid rgba(42,45,53,0.3);align-items:center;font-size:11px}
.result-row:last-child{border-bottom:none}
.score-bar{width:60px;height:5px;background:rgba(0,0,0,0.2);border-radius:3px;overflow:hidden}
.score-fill{height:100%;border-radius:3px}
.benchmark-grid{display:grid;grid-template-columns:1fr 1fr;gap:12px;margin-top:8px}
.bench-col{background:rgba(0,0,0,0.2);border-radius:3px;padding:10px}
.bench-label{font-size:10px;text-transform:uppercase;letter-spacing:1px;margin-bottom:6px;font-weight:700}
.threshold-slider{display:flex;align-items:center;gap:8px;padding:0 12px;margin:4px 0}
.threshold-slider input[type=range]{flex:1;accent-color:var(--accent)}
.threshold-slider .val{font-weight:700;min-width:36px;text-align:right}
</style></head><body>
<h1><span>Pipeline Lab</span> // Lakehouse</h1>
<div class="subtitle">Embedding-based screening vs LLM classification &#x2014; iterative experimentation</div>
<div class="status-bar" id="status-bar">
<div class="stat"><span>Exemplars:</span> <b id="st-exemplars">0</b></div>
<div class="stat"><span>Categories:</span> <b id="st-categories">0</b></div>
<div class="stat"><span>Pipelines:</span> <b id="st-pipelines">0</b></div>
<div class="stat" style="margin-left:auto"><span>Sidecar:</span> <b id="st-health" style="color:var(--text2)">...</b></div>
</div>
<div class="top-bar">
<button class="btn primary" onclick="addCell('exemplars')">+ Exemplars</button>
<button class="btn gold" onclick="addCell('screen')">+ Screen</button>
<button class="btn blue" onclick="addCell('classify')">+ Classify</button>
<button class="btn purple" onclick="addCell('benchmark')">+ Benchmark</button>
<button class="btn" onclick="addCell('similarity')">+ Similarity</button>
<button class="btn" onclick="addCell('generate')">+ Generate</button>
<button class="btn" onclick="addCell('pipeline')">+ Pipeline</button>
<span style="flex:1"></span>
<button class="btn red" onclick="clearCells()">Clear All</button>
</div>
<div class="cells" id="cells"></div>
<script>
var BASE = '';
var cellCounter = 0;
function esc(t){var d=document.createElement('span');d.textContent=String(t);return d.innerHTML}
async function api(path, body) {
var opts = body ? {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(body)} : {};
var r = await fetch(BASE + '/lab' + path, opts);
return r.json();
}
async function refreshStatus() {
try {
var ex = await api('/exemplars');
var pl = await api('/pipelines');
var h = await fetch(BASE + '/health').then(function(r){return r.json()});
document.getElementById('st-exemplars').textContent = ex.total || 0;
document.getElementById('st-categories').textContent = Object.keys(ex.categories || {}).length;
document.getElementById('st-pipelines').textContent = (pl.pipelines || []).length;
document.getElementById('st-health').textContent = h.status || 'ok';
document.getElementById('st-health').style.color = 'var(--accent)';
} catch(e) {
document.getElementById('st-health').textContent = 'error';
document.getElementById('st-health').style.color = 'var(--red)';
}
}
function addCell(type) {
var id = 'cell-' + (++cellCounter);
var cells = document.getElementById('cells');
var cell = document.createElement('div'); cell.className = 'cell'; cell.id = id;
var colors = {exemplars:'var(--accent)',screen:'var(--gold)',classify:'var(--blue)',benchmark:'var(--purple)',similarity:'var(--text2)',generate:'var(--text2)',pipeline:'var(--accent)'};
var labels = {exemplars:'EXEMPLARS',screen:'SCREEN',classify:'CLASSIFY (LLM)',benchmark:'BENCHMARK A/B',similarity:'SIMILARITY',generate:'GENERATE',pipeline:'PIPELINE'};
var placeholders = {
exemplars:'Category: decision\n---\nWe decided to use Parquet for all storage\nThe team chose React over Vue\nArchitecture decision: microservices',
screen:'Enter texts to classify via embedding similarity (one per line):\n\nWe decided to migrate to PostgreSQL\nThe weather is nice today\nArchitecture: chose event sourcing over CRUD',
classify:'Enter texts to classify via LLM (one per line):\n\nWe decided to migrate to PostgreSQL\nThe weather is nice today',
benchmark:'Enter texts to benchmark (one per line):\n\nWe decided to use Kubernetes for orchestration\nThe new hire starts Monday\nTechnical debt: refactor the auth module\nLunch menu looks good today',
similarity:'Enter texts to compare pairwise (one per line):\n\nWe chose React for the frontend\nReact was selected as our UI framework\nThe database uses PostgreSQL',
generate:'Enter a prompt for the LLM...',
pipeline:'Pipeline name: my-extraction\n---\nscreen | threshold=0.6\nclassify\nextract | prompt=Extract the key decision and its rationale\nvalidate | dedup_threshold=0.9'
};
var color = colors[type] || 'var(--text2)';
var label = labels[type] || type.toUpperCase();
var ph = placeholders[type] || '';
// Build cell using DOM methods
var header = document.createElement('div'); header.className = 'cell-header';
var typeSpan = document.createElement('span'); typeSpan.className = 'cell-type'; typeSpan.style.color = color; typeSpan.textContent = label; header.appendChild(typeSpan);
var numSpan = document.createElement('span'); numSpan.textContent = 'Cell #' + cellCounter; header.appendChild(numSpan);
var timeSpan = document.createElement('span'); timeSpan.className = 'cell-time'; timeSpan.id = id + '-time'; header.appendChild(timeSpan);
cell.appendChild(header);
var inputDiv = document.createElement('div'); inputDiv.className = 'cell-input';
var textarea = document.createElement('textarea'); textarea.id = id + '-input'; textarea.placeholder = ph; textarea.value = ph;
inputDiv.appendChild(textarea); cell.appendChild(inputDiv);
if (type === 'screen' || type === 'benchmark') {
var slider = document.createElement('div'); slider.className = 'threshold-slider';
var slLabel = document.createElement('span'); slLabel.style.cssText = 'font-size:10px;color:var(--text2)'; slLabel.textContent = 'Threshold:'; slider.appendChild(slLabel);
var range = document.createElement('input'); range.type = 'range'; range.min = '0.3'; range.max = '0.95'; range.step = '0.05'; range.value = '0.65'; range.id = id + '-threshold';
var valSpan = document.createElement('span'); valSpan.className = 'val'; valSpan.textContent = '0.65';
range.oninput = function() { valSpan.textContent = this.value; };
slider.appendChild(range); slider.appendChild(valSpan); cell.appendChild(slider);
}
var outputDiv = document.createElement('div'); outputDiv.className = 'cell-output'; outputDiv.id = id + '-output';
cell.appendChild(outputDiv);
var tb = document.createElement('div'); tb.className = 'toolbar';
var runBtn = document.createElement('button'); runBtn.className = 'btn primary'; runBtn.textContent = 'Run';
runBtn.onclick = function() { runCell(id, type); }; tb.appendChild(runBtn);
var rmBtn = document.createElement('button'); rmBtn.className = 'btn red'; rmBtn.textContent = 'Remove';
rmBtn.onclick = function() { removeCell(id); }; tb.appendChild(rmBtn);
cell.appendChild(tb);
cells.appendChild(cell);
textarea.focus();
return id;
}
function removeCell(id) { var el = document.getElementById(id); if (el) el.remove(); }
function clearCells() { document.getElementById('cells').textContent = ''; cellCounter = 0; }
function parseLines(text) { return text.split('\n').map(function(l){return l.trim()}).filter(function(l){return l && l.charAt(0) !== '#'}); }
async function runCell(id, type) {
var cell = document.getElementById(id);
var input = document.getElementById(id+'-input').value;
var output = document.getElementById(id+'-output');
var timeEl = document.getElementById(id+'-time');
cell.classList.add('running');
output.className = 'cell-output has-data';
output.textContent = 'Running...';
try {
var t0 = performance.now();
var result;
if (type === 'exemplars') {
var parts = input.split('---');
var catLine = (parts[0] || '').trim();
var category = catLine.replace(/^category:\s*/i, '').trim().toLowerCase();
var texts = parseLines(parts.slice(1).join('\n'));
if (!category || !texts.length) { output.textContent = 'Format: Category: name\\n---\\ntext1\\ntext2'; return; }
result = await api('/exemplars', {category: category, texts: texts});
output.textContent = 'Added ' + result.added + ' exemplars to "' + result.category + '" (total: ' + result.total + ')';
output.style.color = 'var(--accent)';
refreshStatus();
}
else if (type === 'screen') {
var texts = parseLines(input);
var threshold = parseFloat((document.getElementById(id+'-threshold') || {}).value || '0.65');
result = await api('/screen', {texts: texts, threshold: threshold});
renderScreenResults(output, result, threshold);
}
else if (type === 'classify') {
var texts = parseLines(input);
result = await api('/classify', {texts: texts});
renderClassifyResults(output, result);
}
else if (type === 'benchmark') {
var texts = parseLines(input);
var threshold = parseFloat((document.getElementById(id+'-threshold') || {}).value || '0.65');
result = await api('/benchmark', {texts: texts, threshold: threshold});
renderBenchmark(output, result);
}
else if (type === 'similarity') {
var texts = parseLines(input);
result = await api('/cell', {action:'similarity', texts: texts});
renderSimilarityMatrix(output, result);
}
else if (type === 'generate') {
result = await api('/cell', {action:'generate', text: input});
output.textContent = result.text || '(empty)';
}
else if (type === 'pipeline') {
var parts = input.split('---');
var nameLine = (parts[0] || '').trim();
var pName = nameLine.replace(/^pipeline\s*name:\s*/i, '').trim();
var stageLines = parseLines(parts.slice(1).join('\n'));
var stages = stageLines.map(function(line) {
var ps = line.split('|').map(function(s){return s.trim()});
var mode = ps[0];
var config = {};
ps.slice(1).forEach(function(p) {
var kv = p.split('='); if (kv.length===2) {
var v = kv[1].trim();
config[kv[0].trim()] = isNaN(parseFloat(v)) ? v : parseFloat(v);
}
});
return {name: mode, mode: mode, config: config};
});
await api('/pipelines', {name: pName, stages: stages, description: 'Created in Pipeline Lab'});
output.textContent = 'Pipeline "' + pName + '" saved (' + stages.length + ' stages). Use the API to run it: POST /lab/pipelines/run';
output.style.color = 'var(--accent)';
refreshStatus();
}
var elapsed = Math.round(performance.now() - t0);
timeEl.textContent = elapsed + 'ms' + (result && result.time_ms ? ' (server: '+result.time_ms+'ms)' : '');
} catch(e) {
output.textContent = 'Error: ' + e.message;
output.style.color = 'var(--red)';
} finally {
cell.classList.remove('running');
}
}
function renderScreenResults(el, results, threshold) {
el.textContent = '';
results.forEach(function(r) {
var row = document.createElement('div'); row.className = 'result-row';
var cat = document.createElement('span');
cat.style.cssText = 'min-width:80px;font-weight:700;color:' + (r.above_threshold ? 'var(--accent)' : 'var(--text2)');
cat.textContent = r.best_category || 'none'; row.appendChild(cat);
var sim = document.createElement('span'); sim.style.cssText = 'min-width:50px;font-weight:700';
sim.textContent = (r.similarity * 100).toFixed(1) + '%';
sim.style.color = r.similarity >= 0.7 ? 'var(--accent)' : r.similarity >= threshold ? 'var(--gold)' : 'var(--text2)';
row.appendChild(sim);
var bar = document.createElement('div'); bar.className = 'score-bar';
var fill = document.createElement('div'); fill.className = 'score-fill';
fill.style.width = (r.similarity * 100) + '%';
fill.style.background = r.similarity >= 0.7 ? 'var(--accent)' : r.similarity >= threshold ? 'var(--gold)' : 'var(--red)';
bar.appendChild(fill); row.appendChild(bar);
var txt = document.createElement('span'); txt.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
txt.textContent = r.text; row.appendChild(txt);
var badge = document.createElement('span');
badge.style.cssText = 'font-size:9px;padding:2px 6px;border-radius:2px;border:1px solid;' +
(r.above_threshold ? 'color:var(--accent);border-color:var(--accent)' : 'color:var(--text2);border-color:var(--border)');
badge.textContent = r.above_threshold ? 'PASS' : 'FILTERED'; row.appendChild(badge);
el.appendChild(row);
});
}
function renderClassifyResults(el, results) {
el.textContent = '';
results.forEach(function(r) {
var row = document.createElement('div'); row.className = 'result-row';
var cat = document.createElement('span'); cat.style.cssText = 'min-width:80px;font-weight:700;color:var(--blue)';
cat.textContent = r.category; row.appendChild(cat);
var conf = document.createElement('span');
conf.style.cssText = 'min-width:50px;font-size:10px;color:' + (r.confidence==='high'?'var(--accent)':r.confidence==='medium'?'var(--gold)':'var(--text2)');
conf.textContent = r.confidence; row.appendChild(conf);
var txt = document.createElement('span'); txt.style.flex = '1'; txt.textContent = r.text; row.appendChild(txt);
el.appendChild(row);
});
}
function renderBenchmark(el, result) {
el.textContent = '';
// Summary stats (using safe DOM construction)
var summary = document.createElement('div'); summary.style.cssText = 'display:flex;gap:16px;margin-bottom:12px;flex-wrap:wrap';
var stats = [
['Agreement', (result.agreement_rate*100).toFixed(1)+'%', result.agreement_rate>=0.8?'var(--accent)':'var(--gold)'],
['Speedup', result.speedup+'x', result.speedup>=2?'var(--accent)':'var(--text)'],
['Embed', result.embed_time_ms+'ms', 'var(--gold)'],
['LLM', result.llm_time_ms+'ms', 'var(--blue)'],
['Hybrid est.', result.hybrid_estimated_ms+'ms', 'var(--accent)'],
['Screened out', result.texts_screened_out+'/'+result.total_texts, 'var(--purple)']
];
stats.forEach(function(s) {
var box = document.createElement('div'); box.style.cssText = 'background:rgba(0,0,0,0.2);padding:6px 10px;border-radius:3px;text-align:center';
var lbl = document.createElement('div'); lbl.style.cssText = 'font-size:9px;color:var(--text2);text-transform:uppercase;letter-spacing:0.5px'; lbl.textContent = s[0]; box.appendChild(lbl);
var val = document.createElement('div'); val.style.cssText = 'font-size:16px;font-weight:700;color:'+s[2]; val.textContent = s[1]; box.appendChild(val);
summary.appendChild(box);
});
el.appendChild(summary);
// Side-by-side comparison
var grid = document.createElement('div'); grid.style.cssText = 'display:grid;grid-template-columns:1fr 1fr;gap:12px;margin-top:8px';
// Embed column
var leftCol = document.createElement('div'); leftCol.style.cssText = 'background:rgba(0,0,0,0.2);border-radius:3px;padding:10px';
var leftTitle = document.createElement('div'); leftTitle.style.cssText = 'font-size:10px;text-transform:uppercase;letter-spacing:1px;margin-bottom:6px;font-weight:700;color:var(--gold)';
leftTitle.textContent = 'EMBEDDING SCREENING (' + result.embed_time_ms + 'ms)'; leftCol.appendChild(leftTitle);
(result.embed_results||[]).forEach(function(r) {
var row = document.createElement('div'); row.style.cssText = 'font-size:11px;padding:3px 0;display:flex;gap:6px;align-items:center';
var c = document.createElement('span'); c.style.cssText = 'min-width:60px;font-weight:700;color:'+(r.above_threshold?'var(--accent)':'var(--text2)'); c.textContent = r.best_category||'none'; row.appendChild(c);
var s = document.createElement('span'); s.style.cssText = 'min-width:40px;color:var(--text2)'; s.textContent = (r.similarity*100).toFixed(0)+'%'; row.appendChild(s);
var t = document.createElement('span'); t.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap'; t.textContent = r.text; row.appendChild(t);
leftCol.appendChild(row);
});
grid.appendChild(leftCol);
// LLM column
var rightCol = document.createElement('div'); rightCol.style.cssText = 'background:rgba(0,0,0,0.2);border-radius:3px;padding:10px';
var rightTitle = document.createElement('div'); rightTitle.style.cssText = 'font-size:10px;text-transform:uppercase;letter-spacing:1px;margin-bottom:6px;font-weight:700;color:var(--blue)';
rightTitle.textContent = 'LLM CLASSIFICATION (' + result.llm_time_ms + 'ms)'; rightCol.appendChild(rightTitle);
(result.llm_results||[]).forEach(function(r) {
var row = document.createElement('div'); row.style.cssText = 'font-size:11px;padding:3px 0;display:flex;gap:6px;align-items:center';
var c = document.createElement('span'); c.style.cssText = 'min-width:60px;font-weight:700;color:var(--blue)'; c.textContent = r.category; row.appendChild(c);
var s = document.createElement('span'); s.style.cssText = 'min-width:40px;color:'+(r.confidence==='high'?'var(--accent)':'var(--text2)'); s.textContent = r.confidence; row.appendChild(s);
var t = document.createElement('span'); t.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap'; t.textContent = r.text; row.appendChild(t);
rightCol.appendChild(row);
});
grid.appendChild(rightCol);
el.appendChild(grid);
}
function renderSimilarityMatrix(el, result) {
el.textContent = '';
var matrix = result.matrix || [];
var texts = result.texts || [];
if (!matrix.length) { el.textContent = 'No results'; return; }
var tbl = document.createElement('table'); tbl.style.cssText = 'border-collapse:collapse;font-size:11px;width:100%';
var hdr = document.createElement('tr');
var corner = document.createElement('th'); hdr.appendChild(corner);
texts.forEach(function(t) {
var th = document.createElement('th'); th.style.cssText = 'padding:4px;color:var(--text2);font-size:9px;max-width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
th.textContent = t.substring(0, 20); th.title = t; hdr.appendChild(th);
});
tbl.appendChild(hdr);
matrix.forEach(function(row, i) {
var tr = document.createElement('tr');
var td0 = document.createElement('td'); td0.style.cssText = 'padding:4px;color:var(--text2);font-size:9px;max-width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap';
td0.textContent = texts[i].substring(0, 20); tr.appendChild(td0);
row.forEach(function(v, j) {
var td = document.createElement('td');
var bg = i===j ? 'rgba(74,222,128,0.1)' : v>=0.8 ? 'rgba(74,222,128,0.15)' : v>=0.6 ? 'rgba(226,181,90,0.1)' : 'transparent';
td.style.cssText = 'padding:4px;text-align:center;font-weight:'+(v>=0.7?'700':'400')+';color:'+(v>=0.8?'var(--accent)':v>=0.6?'var(--gold)':'var(--text2)')+';background:'+bg;
td.textContent = v.toFixed(2); tr.appendChild(td);
});
tbl.appendChild(tr);
});
el.appendChild(tbl);
}
refreshStatus();
</script>
</body></html>"""
@router.get("", response_class=HTMLResponse)
async def lab_page():
return _get_lab_html()
+503
View File
@@ -0,0 +1,503 @@
"""Pipeline Lab — iterative embedding/LLM pipeline experimentation.
Provides:
- Exemplar-based embedding classification (fast screening)
- LLM-based classification (accurate but slow)
- A/B benchmarking between the two
- Pipeline definition and execution
- Notebook-style API for interactive experimentation
"""
import json
import math
import os
import time
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, HTTPException
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from .ollama import client
router = APIRouter()
EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-embed-text")
GEN_MODEL = os.environ.get("GEN_MODEL", "qwen2.5")
LAB_DIR = Path(os.environ.get("LAB_DIR", "./data/_pipeline_lab"))
LAB_DIR.mkdir(parents=True, exist_ok=True)
# ─── Vector math ─────────────────────────────────────────────
def cosine_similarity(a: list[float], b: list[float]) -> float:
dot = sum(x * y for x, y in zip(a, b))
norm_a = math.sqrt(sum(x * x for x in a))
norm_b = math.sqrt(sum(x * x for x in b))
if norm_a == 0 or norm_b == 0:
return 0.0
return dot / (norm_a * norm_b)
# ─── Exemplar store ──────────────────────────────────────────
# Exemplars are labeled text+embedding pairs used for classification.
# e.g. category="decision" texts=["We decided to use Parquet", "The team chose React"]
_exemplars: dict[str, list[dict]] = {} # category -> [{text, embedding}]
def _exemplar_file() -> Path:
return LAB_DIR / "exemplars.json"
def _load_exemplars():
global _exemplars
fp = _exemplar_file()
if fp.exists():
data = json.loads(fp.read_text())
_exemplars = data
return _exemplars
def _save_exemplars():
_exemplar_file().write_text(json.dumps(_exemplars, indent=2))
_load_exemplars()
# ─── Pipeline store ──────────────────────────────────────────
def _pipelines_dir() -> Path:
d = LAB_DIR / "pipelines"
d.mkdir(exist_ok=True)
return d
# ─── Embedding helper ────────────────────────────────────────
async def _embed_texts(texts: list[str], model: str = EMBED_MODEL) -> list[list[float]]:
embeddings = []
async with client() as c:
for text in texts:
resp = await c.post("/api/embed", json={"model": model, "input": text})
if resp.status_code != 200:
raise HTTPException(502, f"Ollama embed error: {resp.text}")
data = resp.json()
embeddings.extend(data.get("embeddings", []))
return embeddings
async def _generate(prompt: str, model: str = GEN_MODEL, temperature: float = 0.3) -> str:
async with client() as c:
resp = await c.post("/api/generate", json={
"model": model, "prompt": prompt, "stream": False,
"options": {"temperature": temperature, "num_predict": 1024}
})
if resp.status_code != 200:
raise HTTPException(502, f"Ollama generate error: {resp.text}")
return resp.json().get("response", "")
# ─── API: Exemplars ──────────────────────────────────────────
class ExemplarAdd(BaseModel):
category: str
texts: list[str]
class ExemplarList(BaseModel):
categories: dict[str, int] # category -> count
@router.post("/exemplars")
async def add_exemplars(req: ExemplarAdd):
"""Add labeled exemplar texts for a category. Embeddings generated automatically."""
category = req.category.strip().lower()
if not category or not req.texts:
raise HTTPException(400, "category and texts required")
embeddings = await _embed_texts(req.texts)
if category not in _exemplars:
_exemplars[category] = []
for text, emb in zip(req.texts, embeddings):
_exemplars[category].append({"text": text, "embedding": emb})
_save_exemplars()
return {"ok": True, "category": category, "added": len(req.texts),
"total": len(_exemplars[category])}
@router.get("/exemplars")
async def list_exemplars():
"""List all exemplar categories and counts."""
return {"categories": {k: len(v) for k, v in _exemplars.items()},
"total": sum(len(v) for v in _exemplars.values())}
@router.delete("/exemplars/{category}")
async def delete_exemplar_category(category: str):
if category in _exemplars:
del _exemplars[category]
_save_exemplars()
return {"ok": True}
# ─── API: Screen (embedding-based classification) ────────────
class ScreenRequest(BaseModel):
texts: list[str]
threshold: float = 0.65
top_k: int = 1
class ScreenResult(BaseModel):
text: str
best_category: str | None
similarity: float
above_threshold: bool
all_scores: dict[str, float]
@router.post("/screen", response_model=list[ScreenResult])
async def screen_texts(req: ScreenRequest):
"""Classify texts by cosine similarity to exemplar embeddings (fast path)."""
if not _exemplars:
raise HTTPException(400, "No exemplars defined. Add exemplars first.")
embeddings = await _embed_texts(req.texts)
results = []
for text, emb in zip(req.texts, embeddings):
category_scores = {}
for category, exemplar_list in _exemplars.items():
sims = [cosine_similarity(emb, ex["embedding"]) for ex in exemplar_list]
category_scores[category] = max(sims) if sims else 0.0
best_cat = max(category_scores, key=category_scores.get) if category_scores else None
best_sim = category_scores.get(best_cat, 0.0) if best_cat else 0.0
results.append(ScreenResult(
text=text[:200],
best_category=best_cat if best_sim >= req.threshold else None,
similarity=round(best_sim, 4),
above_threshold=best_sim >= req.threshold,
all_scores={k: round(v, 4) for k, v in sorted(category_scores.items(),
key=lambda x: x[1], reverse=True)},
))
return results
# ─── API: Classify (LLM-based classification) ────────────────
class ClassifyRequest(BaseModel):
texts: list[str]
categories: list[str] | None = None # if None, use exemplar category names
model: str | None = None
class ClassifyResult(BaseModel):
text: str
category: str
confidence: str
reasoning: str
@router.post("/classify", response_model=list[ClassifyResult])
async def classify_texts(req: ClassifyRequest):
"""Classify texts using LLM (slow but accurate path)."""
categories = req.categories or list(_exemplars.keys())
if not categories:
raise HTTPException(400, "No categories. Provide categories or add exemplars.")
model = req.model or GEN_MODEL
results = []
for text in req.texts:
prompt = (
f"Classify this text into exactly ONE of these categories: {', '.join(categories)}\n\n"
f"TEXT: {text[:500]}\n\n"
f"Respond with JSON: {{\"category\": \"...\", \"confidence\": \"high|medium|low\", "
f"\"reasoning\": \"one sentence\"}}"
)
raw = await _generate(prompt, model=model, temperature=0.1)
# Parse
try:
j_s, j_e = raw.find("{"), raw.rfind("}") + 1
parsed = json.loads(raw[j_s:j_e]) if j_s >= 0 and j_e > j_s else {}
except Exception:
parsed = {}
results.append(ClassifyResult(
text=text[:200],
category=parsed.get("category", "unknown"),
confidence=parsed.get("confidence", "low"),
reasoning=parsed.get("reasoning", raw[:200]),
))
return results
# ─── API: Benchmark (A/B comparison) ─────────────────────────
class BenchmarkRequest(BaseModel):
texts: list[str]
threshold: float = 0.65
model: str | None = None
class BenchmarkResult(BaseModel):
total_texts: int
# Embedding path
embed_time_ms: int
embed_results: list[dict]
# LLM path
llm_time_ms: int
llm_results: list[dict]
# Comparison
agreement_rate: float
speedup: float
texts_screened_out: int
texts_needing_llm: int
hybrid_estimated_ms: int
@router.post("/benchmark", response_model=BenchmarkResult)
async def benchmark(req: BenchmarkRequest):
"""Run same texts through embedding screening and LLM classification. Compare."""
if not _exemplars:
raise HTTPException(400, "No exemplars. Add exemplars first.")
categories = list(_exemplars.keys())
# Embedding path
t0 = time.monotonic()
embed_results = await screen_texts(ScreenRequest(
texts=req.texts, threshold=req.threshold
))
embed_ms = int((time.monotonic() - t0) * 1000)
# LLM path
t0 = time.monotonic()
llm_results = await classify_texts(ClassifyRequest(
texts=req.texts, categories=categories, model=req.model
))
llm_ms = int((time.monotonic() - t0) * 1000)
# Compare
agreements = 0
screened_out = 0
for er, lr in zip(embed_results, llm_results):
if not er.above_threshold:
screened_out += 1
if er.best_category == lr.category:
agreements += 1
needing_llm = len(req.texts) - screened_out
# Hybrid estimate: embed all + LLM only the uncertain ones
per_text_embed_ms = embed_ms / max(len(req.texts), 1)
per_text_llm_ms = llm_ms / max(len(req.texts), 1)
hybrid_ms = int(embed_ms + needing_llm * per_text_llm_ms)
return BenchmarkResult(
total_texts=len(req.texts),
embed_time_ms=embed_ms,
embed_results=[r.model_dump() for r in embed_results],
llm_time_ms=llm_ms,
llm_results=[r.model_dump() for r in llm_results],
agreement_rate=round(agreements / max(len(req.texts), 1), 3),
speedup=round(llm_ms / max(hybrid_ms, 1), 2),
texts_screened_out=screened_out,
texts_needing_llm=needing_llm,
hybrid_estimated_ms=hybrid_ms,
)
# ─── API: Pipeline definition & execution ────────────────────
class PipelineStage(BaseModel):
name: str
mode: str # "screen", "classify", "extract", "validate", "custom"
config: dict = {} # stage-specific config (threshold, prompt, etc.)
class PipelineDef(BaseModel):
name: str
stages: list[PipelineStage]
description: str = ""
class PipelineRunRequest(BaseModel):
pipeline_name: str
texts: list[str]
@router.post("/pipelines")
async def save_pipeline(pipeline: PipelineDef):
"""Save a pipeline definition."""
fp = _pipelines_dir() / f"{pipeline.name}.json"
fp.write_text(pipeline.model_dump_json(indent=2))
return {"ok": True, "name": pipeline.name}
@router.get("/pipelines")
async def list_pipelines():
"""List saved pipeline definitions."""
pipelines = []
for fp in _pipelines_dir().glob("*.json"):
try:
data = json.loads(fp.read_text())
pipelines.append({"name": data["name"], "stages": len(data["stages"]),
"description": data.get("description", "")})
except Exception:
pass
return {"pipelines": pipelines}
@router.get("/pipelines/{name}")
async def get_pipeline(name: str):
fp = _pipelines_dir() / f"{name}.json"
if not fp.exists():
raise HTTPException(404, "Pipeline not found")
return json.loads(fp.read_text())
@router.post("/pipelines/run")
async def run_pipeline(req: PipelineRunRequest):
"""Execute a pipeline on a set of texts. Returns per-stage results and timing."""
fp = _pipelines_dir() / f"{req.pipeline_name}.json"
if not fp.exists():
raise HTTPException(404, f"Pipeline '{req.pipeline_name}' not found")
pipeline = json.loads(fp.read_text())
results = {"pipeline": req.pipeline_name, "stages": [], "total_ms": 0}
current_texts = req.texts[:]
for stage_def in pipeline["stages"]:
stage_name = stage_def["name"]
mode = stage_def["mode"]
config = stage_def.get("config", {})
t0 = time.monotonic()
stage_result = {"name": stage_name, "mode": mode, "input_count": len(current_texts)}
if mode == "screen":
threshold = config.get("threshold", 0.65)
screen_res = await screen_texts(ScreenRequest(
texts=current_texts, threshold=threshold
))
passed = [r for r in screen_res if r.above_threshold]
stage_result["output_count"] = len(passed)
stage_result["filtered_out"] = len(current_texts) - len(passed)
stage_result["results"] = [r.model_dump() for r in screen_res]
# Pass only above-threshold texts to next stage
current_texts = [r.text for r in screen_res if r.above_threshold]
elif mode == "classify":
cls_res = await classify_texts(ClassifyRequest(
texts=current_texts,
categories=config.get("categories"),
model=config.get("model"),
))
stage_result["output_count"] = len(cls_res)
stage_result["results"] = [r.model_dump() for r in cls_res]
elif mode == "extract":
extract_prompt = config.get("prompt", "Extract key information from this text:")
extractions = []
for text in current_texts:
raw = await _generate(f"{extract_prompt}\n\nTEXT: {text[:800]}")
extractions.append({"text": text[:200], "extracted": raw})
stage_result["output_count"] = len(extractions)
stage_result["results"] = extractions
elif mode == "validate":
# Embedding-based dedup: find near-duplicate results
if len(current_texts) > 1:
embs = await _embed_texts(current_texts)
dupes = []
threshold = config.get("dedup_threshold", 0.92)
for i in range(len(embs)):
for j in range(i + 1, len(embs)):
sim = cosine_similarity(embs[i], embs[j])
if sim >= threshold:
dupes.append({"i": i, "j": j, "similarity": round(sim, 4),
"text_a": current_texts[i][:100],
"text_b": current_texts[j][:100]})
stage_result["duplicates_found"] = len(dupes)
stage_result["results"] = dupes
else:
stage_result["duplicates_found"] = 0
stage_result["results"] = []
stage_result["output_count"] = len(current_texts)
else:
stage_result["error"] = f"Unknown mode: {mode}"
stage_result["output_count"] = len(current_texts)
stage_ms = int((time.monotonic() - t0) * 1000)
stage_result["time_ms"] = stage_ms
results["stages"].append(stage_result)
results["total_ms"] += stage_ms
return results
# ─── API: REPL cell (free-form eval) ─────────────────────────
class CellRequest(BaseModel):
action: str # "embed", "generate", "similarity", "screen", "classify"
text: str = ""
texts: list[str] = []
params: dict = {}
@router.post("/cell")
async def run_cell(req: CellRequest):
"""Execute a single notebook cell. Flexible entry point for ad-hoc operations."""
t0 = time.monotonic()
result = {}
if req.action == "embed":
texts = req.texts or ([req.text] if req.text else [])
embs = await _embed_texts(texts)
result = {"embeddings_count": len(embs), "dimensions": len(embs[0]) if embs else 0,
"texts": texts}
elif req.action == "generate":
text = await _generate(req.text, **{k: v for k, v in req.params.items()
if k in ("model", "temperature")})
result = {"text": text}
elif req.action == "similarity":
if len(req.texts) < 2:
raise HTTPException(400, "Need at least 2 texts for similarity")
embs = await _embed_texts(req.texts)
matrix = []
for i in range(len(embs)):
row = []
for j in range(len(embs)):
row.append(round(cosine_similarity(embs[i], embs[j]), 4))
matrix.append(row)
result = {"matrix": matrix, "texts": [t[:80] for t in req.texts]}
elif req.action == "screen":
texts = req.texts or ([req.text] if req.text else [])
threshold = req.params.get("threshold", 0.65)
res = await screen_texts(ScreenRequest(texts=texts, threshold=threshold))
result = {"results": [r.model_dump() for r in res]}
elif req.action == "classify":
texts = req.texts or ([req.text] if req.text else [])
res = await classify_texts(ClassifyRequest(texts=texts))
result = {"results": [r.model_dump() for r in res]}
else:
raise HTTPException(400, f"Unknown action: {req.action}")
result["time_ms"] = int((time.monotonic() - t0) * 1000)
return result