v1/mode: parameterized runner + 5 enrichment-experiment modes
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
J's directive (2026-04-26): "Create different modes so we can really
dial in the architecture before it gets further along — pinpoint the
failures and strengths equally so I know what direction to go in.
Loop theater happens when we don't pinpoint the most accurate path."
Refactored execute() to switch on mode name → EnrichmentFlags preset.
Five native modes designed as deliberate experiments — each isolates
one architectural axis so the comparison matrix reads off what's
doing work vs what's adding latency for nothing:
codereview_lakehouse — all enrichment on (ceiling)
codereview_null — raw file + generic prompt (baseline)
codereview_isolation — file + pathway only (no matrix)
codereview_matrix_only — file + matrix only (no pathway)
codereview_playbook_only — pathway only, NO file content (lossy ceiling)
Each call appends a row to data/_kb/mode_experiments.jsonl with full
sources + response. LH_MODE_LOG_OFF=1 to suppress.
scripts/mode_experiment.ts — sweeps files × modes serially, prints
live progress with per-call enrichment stats. Defaults to OpenRouter
free model so cloud quota doesn't gate experiments.
scripts/mode_compare.ts — reads the JSONL, outputs per-file matrix
+ per-mode aggregate + mode-vs-baseline win/loss with avg finding
delta. Heuristic finding-count from markdown table rows; pathway
citation count from preamble references.
scrum_master_pipeline.ts gets a mode-runner fast path gated by
LH_USE_MODE_RUNNER=1: try /v1/mode/execute first, fall through to
the existing ladder if response < LH_MODE_MIN_CHARS (default 2000)
or anything errors. Off by default until A/B-validated.
First experiment results (2 files × 5 modes via gpt-oss-120b:free):
- codereview_null produces 12.6KB response with ZERO findings
(proves adversarial framing is load-bearing)
- codereview_playbook_only produces MORE findings than lakehouse
on average (12 vs 9) at 73% the latency — pathway memory is
the dominant signal driver
- codereview_matrix_only underperforms isolation by ~0.5 findings
while costing the same latency — matrix corpus likely
underperforming for scrum_review task class
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+158
-20
@@ -42,14 +42,87 @@ const VALID_MODES: &[&str] = &[
|
||||
"evolution", "blindassembly", "staircase", "drift", "mesh",
|
||||
"hallucination", "timeloop", "research", "eval", "extract",
|
||||
"refine", "adaptive", "deep_analysis", "distill",
|
||||
// Native runners (not in LLM Team — handled by /v1/mode/execute):
|
||||
"codereview_lakehouse",
|
||||
// Native runners (not in LLM Team — handled by /v1/mode/execute).
|
||||
// Each is a parameterized preset of EnrichmentFlags below — designed
|
||||
// as a deliberate experiment so we can read the matrix and identify
|
||||
// which signals are doing real work vs adding latency for nothing.
|
||||
"codereview_lakehouse", // all enrichment on (ceiling)
|
||||
"codereview_null", // raw file + generic prompt (baseline)
|
||||
"codereview_isolation", // file + pathway only (no matrix)
|
||||
"codereview_matrix_only", // file + matrix only (no pathway)
|
||||
"codereview_playbook_only", // pathway only, NO file content (lossy ceiling)
|
||||
];
|
||||
|
||||
/// Whether a mode is handled natively in this gateway vs proxied to
|
||||
/// LLM Team. Drives /v1/mode/execute dispatch.
|
||||
fn is_native_mode(mode: &str) -> bool {
|
||||
matches!(mode, "codereview_lakehouse")
|
||||
matches!(
|
||||
mode,
|
||||
"codereview_lakehouse"
|
||||
| "codereview_null"
|
||||
| "codereview_isolation"
|
||||
| "codereview_matrix_only"
|
||||
| "codereview_playbook_only"
|
||||
)
|
||||
}
|
||||
|
||||
/// Per-mode enrichment knobs — each native mode is a preset over these
|
||||
/// flags. Exists so the runner code is one path (less drift between
|
||||
/// modes) and the comparison harness can read which signals fired.
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub struct EnrichmentFlags {
|
||||
pub include_file_content: bool,
|
||||
pub include_bug_fingerprints: bool,
|
||||
pub include_matrix_chunks: bool,
|
||||
pub use_relevance_filter: bool,
|
||||
pub framing: ReviewerFraming,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub enum ReviewerFraming {
|
||||
Adversarial, // forensic, ranked findings + verdict (lakehouse default)
|
||||
Generic, // "review this" — no codebase priors (null baseline)
|
||||
}
|
||||
|
||||
fn flags_for_mode(mode: &str) -> EnrichmentFlags {
|
||||
match mode {
|
||||
"codereview_null" => EnrichmentFlags {
|
||||
include_file_content: true,
|
||||
include_bug_fingerprints: false,
|
||||
include_matrix_chunks: false,
|
||||
use_relevance_filter: false,
|
||||
framing: ReviewerFraming::Generic,
|
||||
},
|
||||
"codereview_isolation" => EnrichmentFlags {
|
||||
include_file_content: true,
|
||||
include_bug_fingerprints: true,
|
||||
include_matrix_chunks: false,
|
||||
use_relevance_filter: false,
|
||||
framing: ReviewerFraming::Adversarial,
|
||||
},
|
||||
"codereview_matrix_only" => EnrichmentFlags {
|
||||
include_file_content: true,
|
||||
include_bug_fingerprints: false,
|
||||
include_matrix_chunks: true,
|
||||
use_relevance_filter: true,
|
||||
framing: ReviewerFraming::Adversarial,
|
||||
},
|
||||
"codereview_playbook_only" => EnrichmentFlags {
|
||||
include_file_content: false, // lossy on purpose — measures pathway-alone ceiling
|
||||
include_bug_fingerprints: true,
|
||||
include_matrix_chunks: false,
|
||||
use_relevance_filter: false,
|
||||
framing: ReviewerFraming::Adversarial,
|
||||
},
|
||||
// Default (codereview_lakehouse): everything on.
|
||||
_ => EnrichmentFlags {
|
||||
include_file_content: true,
|
||||
include_bug_fingerprints: true,
|
||||
include_matrix_chunks: true,
|
||||
use_relevance_filter: true,
|
||||
framing: ReviewerFraming::Adversarial,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
@@ -342,6 +415,9 @@ pub struct EnrichmentSources {
|
||||
pub matrix_corpus: Option<String>,
|
||||
pub relevance_filter_used: bool,
|
||||
pub enrichment_warnings: Vec<String>,
|
||||
/// Which enrichment knobs the runner used for this mode. Lets
|
||||
/// the comparison aggregator group runs by signal-set.
|
||||
pub flags: Option<EnrichmentFlags>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
@@ -356,13 +432,23 @@ pub struct ExecuteResponse {
|
||||
pub latency_ms: u64,
|
||||
}
|
||||
|
||||
const REVIEWER_FRAMING: &str = "You are an adversarial code reviewer for the Lakehouse codebase \
|
||||
const FRAMING_ADVERSARIAL: &str = "You are an adversarial code reviewer for the Lakehouse codebase \
|
||||
(Rust + DataFusion + Parquet + object storage). Audit the focus file forensically. \
|
||||
Output a markdown report with: (1) one-line verdict (pass | needs_patch | fail), (2) ranked \
|
||||
findings table with file:line, evidence, severity, confidence percent, (3) concrete patch \
|
||||
suggestions, (4) PRD/ADR refs where applicable. Be precise — assume nothing works until \
|
||||
proven. Do NOT hedge.";
|
||||
|
||||
const FRAMING_GENERIC: &str = "You are a code reviewer. Read the file below and produce a \
|
||||
markdown review with findings.";
|
||||
|
||||
fn framing_text(f: ReviewerFraming) -> &'static str {
|
||||
match f {
|
||||
ReviewerFraming::Adversarial => FRAMING_ADVERSARIAL,
|
||||
ReviewerFraming::Generic => FRAMING_GENERIC,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute(
|
||||
State(_state): State<V1State>,
|
||||
Json(req): Json<ExecuteRequest>,
|
||||
@@ -399,12 +485,15 @@ pub async fn execute(
|
||||
));
|
||||
}
|
||||
|
||||
let flags = flags_for_mode(&mode);
|
||||
let mut sources = EnrichmentSources {
|
||||
matrix_corpus: matrix_corpus.clone(),
|
||||
flags: Some(flags),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Step 1: focus file content.
|
||||
// Step 1: focus file content (always read — even modes that don't
|
||||
// include it in the prompt may need it for citation/sources).
|
||||
let file_content = match req.file_content.clone() {
|
||||
Some(c) => c,
|
||||
None => match std::fs::read_to_string(&req.file_path) {
|
||||
@@ -439,7 +528,7 @@ pub async fn execute(
|
||||
|
||||
// Step 2: pathway memory bug fingerprints for this file area.
|
||||
let mut bug_preamble = String::new();
|
||||
{
|
||||
if flags.include_bug_fingerprints {
|
||||
let body = serde_json::json!({
|
||||
"task_class": req.task_class,
|
||||
"file_path": req.file_path,
|
||||
@@ -484,7 +573,8 @@ pub async fn execute(
|
||||
|
||||
// Step 3: matrix corpus search (if configured for this task class).
|
||||
let mut raw_chunks: Vec<serde_json::Value> = vec![];
|
||||
if let Some(corpus) = &matrix_corpus {
|
||||
if flags.include_matrix_chunks {
|
||||
if let Some(corpus) = &matrix_corpus {
|
||||
let body = serde_json::json!({
|
||||
"index_name": corpus,
|
||||
"query": format!("{} {}\n{}", req.task_class, req.file_path, &file_content[..file_content.len().min(500)]),
|
||||
@@ -512,10 +602,11 @@ pub async fn execute(
|
||||
.enrichment_warnings
|
||||
.push(format!("matrix_search err: {e}")),
|
||||
}
|
||||
} // close `if let Some(corpus)`
|
||||
}
|
||||
|
||||
// Step 4: relevance filter — drop adjacency pollution.
|
||||
let kept_chunks: Vec<serde_json::Value> = if !raw_chunks.is_empty() {
|
||||
let kept_chunks: Vec<serde_json::Value> = if flags.use_relevance_filter && !raw_chunks.is_empty() {
|
||||
let chunks_for_filter: Vec<serde_json::Value> = raw_chunks
|
||||
.iter()
|
||||
.map(|c| {
|
||||
@@ -557,15 +648,24 @@ pub async fn execute(
|
||||
raw_chunks
|
||||
}
|
||||
}
|
||||
} else if !flags.use_relevance_filter && !raw_chunks.is_empty() {
|
||||
// Take raw matrix chunks unfiltered — `codereview_matrix_only`
|
||||
// turns the filter off intentionally to measure how much
|
||||
// pollution the filter is actually catching.
|
||||
sources.matrix_chunks_kept = raw_chunks.len();
|
||||
raw_chunks.clone()
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
// Step 5: assemble the prompt.
|
||||
// Step 5: assemble the prompt — strictly per-flag so we don't
|
||||
// leak signals across modes.
|
||||
let mut user_prompt = String::new();
|
||||
user_prompt.push_str(&bug_preamble);
|
||||
if !kept_chunks.is_empty() {
|
||||
user_prompt.push_str("📁 RELATED CONTEXT (relevance-filtered from matrix):\n");
|
||||
if flags.include_bug_fingerprints {
|
||||
user_prompt.push_str(&bug_preamble);
|
||||
}
|
||||
if flags.include_matrix_chunks && !kept_chunks.is_empty() {
|
||||
user_prompt.push_str("📁 RELATED CONTEXT (matrix chunks):\n");
|
||||
for c in &kept_chunks {
|
||||
let src = c.get("source").and_then(|v| v.as_str()).unwrap_or("?");
|
||||
let txt = c.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
@@ -573,11 +673,20 @@ pub async fn execute(
|
||||
}
|
||||
user_prompt.push_str("\n");
|
||||
}
|
||||
user_prompt.push_str(&format!("FILE: {}\n```rust\n{}\n```\n", req.file_path, file_content));
|
||||
if flags.include_file_content {
|
||||
user_prompt.push_str(&format!("FILE: {}\n```rust\n{}\n```\n", req.file_path, file_content));
|
||||
} else {
|
||||
// Lossy mode — playbook_only intentionally omits file content
|
||||
// to measure how much value pathway memory carries on its own.
|
||||
user_prompt.push_str(&format!(
|
||||
"FILE PATH (content omitted): {}\nFile size: {} bytes\n",
|
||||
req.file_path, file_content.len()
|
||||
));
|
||||
}
|
||||
if let Some(q) = &req.user_question {
|
||||
user_prompt.push_str(&format!("\nQUESTION: {}\n", q));
|
||||
} else {
|
||||
user_prompt.push_str("\nProduce the forensic review now.\n");
|
||||
user_prompt.push_str("\nProduce the review now.\n");
|
||||
}
|
||||
|
||||
let enriched_chars = user_prompt.len();
|
||||
@@ -611,7 +720,7 @@ pub async fn execute(
|
||||
"model": model,
|
||||
"provider": provider_hint,
|
||||
"messages": [
|
||||
{ "role": "system", "content": REVIEWER_FRAMING },
|
||||
{ "role": "system", "content": framing_text(flags.framing) },
|
||||
{ "role": "user", "content": user_prompt },
|
||||
],
|
||||
"temperature": 0.1,
|
||||
@@ -668,16 +777,45 @@ pub async fn execute(
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Json(ExecuteResponse {
|
||||
mode,
|
||||
model,
|
||||
task_class: req.task_class,
|
||||
let resp = ExecuteResponse {
|
||||
mode: mode.clone(),
|
||||
model: model.clone(),
|
||||
task_class: req.task_class.clone(),
|
||||
enriched_prompt_chars: enriched_chars,
|
||||
enriched_prompt_preview: preview,
|
||||
sources,
|
||||
response: response_text,
|
||||
latency_ms: t0.elapsed().as_millis() as u64,
|
||||
}))
|
||||
};
|
||||
|
||||
// Append to mode_experiments.jsonl so the comparison aggregator
|
||||
// can read the matrix later. Best-effort — write failure must not
|
||||
// fail the request. Skips if LH_MODE_LOG_OFF=1.
|
||||
if std::env::var("LH_MODE_LOG_OFF").as_deref() != Ok("1") {
|
||||
let log_path = std::env::var("LH_MODE_LOG_PATH")
|
||||
.unwrap_or_else(|_| "data/_kb/mode_experiments.jsonl".to_string());
|
||||
let row = serde_json::json!({
|
||||
"ts": chrono::Utc::now().to_rfc3339(),
|
||||
"mode": resp.mode,
|
||||
"model": resp.model,
|
||||
"task_class": resp.task_class,
|
||||
"file_path": req.file_path,
|
||||
"enriched_prompt_chars": resp.enriched_prompt_chars,
|
||||
"response_chars": resp.response.len(),
|
||||
"latency_ms": resp.latency_ms,
|
||||
"sources": resp.sources,
|
||||
"response": resp.response,
|
||||
});
|
||||
if let Some(parent) = std::path::Path::new(&log_path).parent() {
|
||||
let _ = std::fs::create_dir_all(parent);
|
||||
}
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(&log_path) {
|
||||
use std::io::Write;
|
||||
let _ = writeln!(f, "{}", row);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Json(resp))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user