v1/mode: parameterized runner + 5 enrichment-experiment modes
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts

J's directive (2026-04-26): "Create different modes so we can really
dial in the architecture before it gets further along — pinpoint the
failures and strengths equally so I know what direction to go in.
Loop theater happens when we don't pinpoint the most accurate path."

Refactored execute() to switch on mode name → EnrichmentFlags preset.
Five native modes designed as deliberate experiments — each isolates
one architectural axis so the comparison matrix reads off what's
doing work vs what's adding latency for nothing:

  codereview_lakehouse     — all enrichment on (ceiling)
  codereview_null          — raw file + generic prompt (baseline)
  codereview_isolation     — file + pathway only (no matrix)
  codereview_matrix_only   — file + matrix only (no pathway)
  codereview_playbook_only — pathway only, NO file content (lossy ceiling)

Each call appends a row to data/_kb/mode_experiments.jsonl with full
sources + response. LH_MODE_LOG_OFF=1 to suppress.

scripts/mode_experiment.ts — sweeps files × modes serially, prints
live progress with per-call enrichment stats. Defaults to OpenRouter
free model so cloud quota doesn't gate experiments.

scripts/mode_compare.ts — reads the JSONL, outputs per-file matrix
+ per-mode aggregate + mode-vs-baseline win/loss with avg finding
delta. Heuristic finding-count from markdown table rows; pathway
citation count from preamble references.

scrum_master_pipeline.ts gets a mode-runner fast path gated by
LH_USE_MODE_RUNNER=1: try /v1/mode/execute first, fall through to
the existing ladder if response < LH_MODE_MIN_CHARS (default 2000)
or anything errors. Off by default until A/B-validated.

First experiment results (2 files × 5 modes via gpt-oss-120b:free):
  - codereview_null produces 12.6KB response with ZERO findings
    (proves adversarial framing is load-bearing)
  - codereview_playbook_only produces MORE findings than lakehouse
    on average (12 vs 9) at 73% the latency — pathway memory is
    the dominant signal driver
  - codereview_matrix_only underperforms isolation by ~0.5 findings
    while costing the same latency — matrix corpus likely
    underperforming for scrum_review task class

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-26 01:36:42 -05:00
co-authored by Claude Opus 4.7
parent 86f63a083d
commit 7c47734287
5 changed files with 526 additions and 20 deletions
+158 -20
View File
@@ -42,14 +42,87 @@ const VALID_MODES: &[&str] = &[
"evolution", "blindassembly", "staircase", "drift", "mesh",
"hallucination", "timeloop", "research", "eval", "extract",
"refine", "adaptive", "deep_analysis", "distill",
// Native runners (not in LLM Team — handled by /v1/mode/execute):
"codereview_lakehouse",
// Native runners (not in LLM Team — handled by /v1/mode/execute).
// Each is a parameterized preset of EnrichmentFlags below — designed
// as a deliberate experiment so we can read the matrix and identify
// which signals are doing real work vs adding latency for nothing.
"codereview_lakehouse", // all enrichment on (ceiling)
"codereview_null", // raw file + generic prompt (baseline)
"codereview_isolation", // file + pathway only (no matrix)
"codereview_matrix_only", // file + matrix only (no pathway)
"codereview_playbook_only", // pathway only, NO file content (lossy ceiling)
];
/// Whether a mode is handled natively in this gateway vs proxied to
/// LLM Team. Drives /v1/mode/execute dispatch.
fn is_native_mode(mode: &str) -> bool {
matches!(mode, "codereview_lakehouse")
matches!(
mode,
"codereview_lakehouse"
| "codereview_null"
| "codereview_isolation"
| "codereview_matrix_only"
| "codereview_playbook_only"
)
}
/// Per-mode enrichment knobs — each native mode is a preset over these
/// flags. Exists so the runner code is one path (less drift between
/// modes) and the comparison harness can read which signals fired.
#[derive(Debug, Clone, Copy, Serialize)]
pub struct EnrichmentFlags {
pub include_file_content: bool,
pub include_bug_fingerprints: bool,
pub include_matrix_chunks: bool,
pub use_relevance_filter: bool,
pub framing: ReviewerFraming,
}
#[derive(Debug, Clone, Copy, Serialize)]
pub enum ReviewerFraming {
Adversarial, // forensic, ranked findings + verdict (lakehouse default)
Generic, // "review this" — no codebase priors (null baseline)
}
fn flags_for_mode(mode: &str) -> EnrichmentFlags {
match mode {
"codereview_null" => EnrichmentFlags {
include_file_content: true,
include_bug_fingerprints: false,
include_matrix_chunks: false,
use_relevance_filter: false,
framing: ReviewerFraming::Generic,
},
"codereview_isolation" => EnrichmentFlags {
include_file_content: true,
include_bug_fingerprints: true,
include_matrix_chunks: false,
use_relevance_filter: false,
framing: ReviewerFraming::Adversarial,
},
"codereview_matrix_only" => EnrichmentFlags {
include_file_content: true,
include_bug_fingerprints: false,
include_matrix_chunks: true,
use_relevance_filter: true,
framing: ReviewerFraming::Adversarial,
},
"codereview_playbook_only" => EnrichmentFlags {
include_file_content: false, // lossy on purpose — measures pathway-alone ceiling
include_bug_fingerprints: true,
include_matrix_chunks: false,
use_relevance_filter: false,
framing: ReviewerFraming::Adversarial,
},
// Default (codereview_lakehouse): everything on.
_ => EnrichmentFlags {
include_file_content: true,
include_bug_fingerprints: true,
include_matrix_chunks: true,
use_relevance_filter: true,
framing: ReviewerFraming::Adversarial,
},
}
}
#[derive(Clone, Debug, Deserialize)]
@@ -342,6 +415,9 @@ pub struct EnrichmentSources {
pub matrix_corpus: Option<String>,
pub relevance_filter_used: bool,
pub enrichment_warnings: Vec<String>,
/// Which enrichment knobs the runner used for this mode. Lets
/// the comparison aggregator group runs by signal-set.
pub flags: Option<EnrichmentFlags>,
}
#[derive(Serialize, Debug)]
@@ -356,13 +432,23 @@ pub struct ExecuteResponse {
pub latency_ms: u64,
}
const REVIEWER_FRAMING: &str = "You are an adversarial code reviewer for the Lakehouse codebase \
const FRAMING_ADVERSARIAL: &str = "You are an adversarial code reviewer for the Lakehouse codebase \
(Rust + DataFusion + Parquet + object storage). Audit the focus file forensically. \
Output a markdown report with: (1) one-line verdict (pass | needs_patch | fail), (2) ranked \
findings table with file:line, evidence, severity, confidence percent, (3) concrete patch \
suggestions, (4) PRD/ADR refs where applicable. Be precise — assume nothing works until \
proven. Do NOT hedge.";
const FRAMING_GENERIC: &str = "You are a code reviewer. Read the file below and produce a \
markdown review with findings.";
fn framing_text(f: ReviewerFraming) -> &'static str {
match f {
ReviewerFraming::Adversarial => FRAMING_ADVERSARIAL,
ReviewerFraming::Generic => FRAMING_GENERIC,
}
}
pub async fn execute(
State(_state): State<V1State>,
Json(req): Json<ExecuteRequest>,
@@ -399,12 +485,15 @@ pub async fn execute(
));
}
let flags = flags_for_mode(&mode);
let mut sources = EnrichmentSources {
matrix_corpus: matrix_corpus.clone(),
flags: Some(flags),
..Default::default()
};
// Step 1: focus file content.
// Step 1: focus file content (always read — even modes that don't
// include it in the prompt may need it for citation/sources).
let file_content = match req.file_content.clone() {
Some(c) => c,
None => match std::fs::read_to_string(&req.file_path) {
@@ -439,7 +528,7 @@ pub async fn execute(
// Step 2: pathway memory bug fingerprints for this file area.
let mut bug_preamble = String::new();
{
if flags.include_bug_fingerprints {
let body = serde_json::json!({
"task_class": req.task_class,
"file_path": req.file_path,
@@ -484,7 +573,8 @@ pub async fn execute(
// Step 3: matrix corpus search (if configured for this task class).
let mut raw_chunks: Vec<serde_json::Value> = vec![];
if let Some(corpus) = &matrix_corpus {
if flags.include_matrix_chunks {
if let Some(corpus) = &matrix_corpus {
let body = serde_json::json!({
"index_name": corpus,
"query": format!("{} {}\n{}", req.task_class, req.file_path, &file_content[..file_content.len().min(500)]),
@@ -512,10 +602,11 @@ pub async fn execute(
.enrichment_warnings
.push(format!("matrix_search err: {e}")),
}
} // close `if let Some(corpus)`
}
// Step 4: relevance filter — drop adjacency pollution.
let kept_chunks: Vec<serde_json::Value> = if !raw_chunks.is_empty() {
let kept_chunks: Vec<serde_json::Value> = if flags.use_relevance_filter && !raw_chunks.is_empty() {
let chunks_for_filter: Vec<serde_json::Value> = raw_chunks
.iter()
.map(|c| {
@@ -557,15 +648,24 @@ pub async fn execute(
raw_chunks
}
}
} else if !flags.use_relevance_filter && !raw_chunks.is_empty() {
// Take raw matrix chunks unfiltered — `codereview_matrix_only`
// turns the filter off intentionally to measure how much
// pollution the filter is actually catching.
sources.matrix_chunks_kept = raw_chunks.len();
raw_chunks.clone()
} else {
vec![]
};
// Step 5: assemble the prompt.
// Step 5: assemble the prompt — strictly per-flag so we don't
// leak signals across modes.
let mut user_prompt = String::new();
user_prompt.push_str(&bug_preamble);
if !kept_chunks.is_empty() {
user_prompt.push_str("📁 RELATED CONTEXT (relevance-filtered from matrix):\n");
if flags.include_bug_fingerprints {
user_prompt.push_str(&bug_preamble);
}
if flags.include_matrix_chunks && !kept_chunks.is_empty() {
user_prompt.push_str("📁 RELATED CONTEXT (matrix chunks):\n");
for c in &kept_chunks {
let src = c.get("source").and_then(|v| v.as_str()).unwrap_or("?");
let txt = c.get("text").and_then(|v| v.as_str()).unwrap_or("");
@@ -573,11 +673,20 @@ pub async fn execute(
}
user_prompt.push_str("\n");
}
user_prompt.push_str(&format!("FILE: {}\n```rust\n{}\n```\n", req.file_path, file_content));
if flags.include_file_content {
user_prompt.push_str(&format!("FILE: {}\n```rust\n{}\n```\n", req.file_path, file_content));
} else {
// Lossy mode — playbook_only intentionally omits file content
// to measure how much value pathway memory carries on its own.
user_prompt.push_str(&format!(
"FILE PATH (content omitted): {}\nFile size: {} bytes\n",
req.file_path, file_content.len()
));
}
if let Some(q) = &req.user_question {
user_prompt.push_str(&format!("\nQUESTION: {}\n", q));
} else {
user_prompt.push_str("\nProduce the forensic review now.\n");
user_prompt.push_str("\nProduce the review now.\n");
}
let enriched_chars = user_prompt.len();
@@ -611,7 +720,7 @@ pub async fn execute(
"model": model,
"provider": provider_hint,
"messages": [
{ "role": "system", "content": REVIEWER_FRAMING },
{ "role": "system", "content": framing_text(flags.framing) },
{ "role": "user", "content": user_prompt },
],
"temperature": 0.1,
@@ -668,16 +777,45 @@ pub async fn execute(
}
};
Ok(Json(ExecuteResponse {
mode,
model,
task_class: req.task_class,
let resp = ExecuteResponse {
mode: mode.clone(),
model: model.clone(),
task_class: req.task_class.clone(),
enriched_prompt_chars: enriched_chars,
enriched_prompt_preview: preview,
sources,
response: response_text,
latency_ms: t0.elapsed().as_millis() as u64,
}))
};
// Append to mode_experiments.jsonl so the comparison aggregator
// can read the matrix later. Best-effort — write failure must not
// fail the request. Skips if LH_MODE_LOG_OFF=1.
if std::env::var("LH_MODE_LOG_OFF").as_deref() != Ok("1") {
let log_path = std::env::var("LH_MODE_LOG_PATH")
.unwrap_or_else(|_| "data/_kb/mode_experiments.jsonl".to_string());
let row = serde_json::json!({
"ts": chrono::Utc::now().to_rfc3339(),
"mode": resp.mode,
"model": resp.model,
"task_class": resp.task_class,
"file_path": req.file_path,
"enriched_prompt_chars": resp.enriched_prompt_chars,
"response_chars": resp.response.len(),
"latency_ms": resp.latency_ms,
"sources": resp.sources,
"response": resp.response,
});
if let Some(parent) = std::path::Path::new(&log_path).parent() {
let _ = std::fs::create_dir_all(parent);
}
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(&log_path) {
use std::io::Write;
let _ = writeln!(f, "{}", row);
}
}
Ok(Json(resp))
}
#[cfg(test)]