distillation: Phase 4 — dataset export layer

Build the contamination firewall: RAG, SFT, and Preference exporters
that turn scored evidence into clean training datasets without
leaking rejected, unvalidated, hallucinated, or provenance-free
records.

Files (8 new + 4 schema updates):
  scripts/distillation/quarantine.ts      shared QuarantineWriter, 11-reason taxonomy
  scripts/distillation/export_rag.ts      RAG exporter (--include-review opt-in)
  scripts/distillation/export_sft.ts      SFT exporter (--include-partial opt-in, SFT_NEVER constant)
  scripts/distillation/export_preference.ts preference exporter, same task_id pairing
  scripts/distillation/distill.ts         CLI dispatcher (build-evidence/score/export-*)
  tests/distillation/exports.test.ts      15 contamination-firewall tests
  reports/distillation/phase4-export-report.md  acceptance report

Schema field-name alignment with now.md:
  rag_sample.ts        +source_category, exported_at→created_at
  sft_sample.ts        +id, exported_at→created_at, partially_accepted at schema (CLI gates)
  preference_sample.ts +id, source_run_ids→chosen_run_id+rejected_run_id, +created_at

Test metrics: 117 distillation tests pass · 0 fail · 315 expects · 327ms

Real-data export run (1052 scored input rows):
  RAG:        446 exported (351 acc + 95 partial), 606 quarantined
  SFT:        351 exported (all 'accepted'),       701 quarantined
  Preference:  83 pairs exported,                   16 quarantined

CONTAMINATION FIREWALL — verified held on real data:
  - SFT output: 351/351 quality_score='accepted' (ZERO leaked)
  - RAG output: 351 acc + 95 partial (ZERO rejected leaked)
  - Preference: 0 self-pairs (chosen_run_id != rejected_run_id)
  - 536 rejected+needs_human_review records caught at unsafe_sft_category
    gate, exact match to scored-runs forbidden-category total

Defense in depth (the firewall is two layers, not one):
  1. Schema layer (Phase 1): SftSample.quality_score enum forbids
     rejected/needs_human at write time
  2. Exporter layer: SFT_NEVER constant in export_sft.ts checks
     category before synthesis. Even if synthesis produced a row
     with quality_score=rejected, validateSftSample would reject it.

Quarantine reasons (11): missing_provenance, missing_source_run_id,
empty_content, schema_violation, unsafe_sft_category,
unsafe_rag_category, invalid_preference_pairing,
hallucinated_file_path, duplicate_id, self_pairing,
category_disallowed.

Bug surfaced + fixed during testing: module-level evidenceCache
shared state across test runs (tests wipe TMP, cache holds stale
empty Map). Moved cache to per-call scope. Same pattern bit Phase 2
materializer would have hit if its tests had multiple runs sharing
state — preventive fix.

Pairing logic v1: same task_id with category gap. accepted×rejected
preferred, accepted×partially_accepted as fallback. MAX_PAIRS_PER_TASK=5
cap prevents one hot task from dominating. Future: cross-source
pairing (scrum_reviews chosen vs observer_reviews rejected on same
file) to grow dataset beyond 83.

CLI: ./scripts/distill.ts {build-evidence|score|export-rag|export-sft|export-preference|export-all|health}
Flags: --dry-run, --include-partial (SFT only), --include-review (RAG only)

Carry-overs to Phase 5 (Receipts Harness):
- Each exporter currently writes results but no per-stage receipt.json.
  Phase 5 wraps build_evidence_index + score_runs + export_* in a
  withReceipt() helper that captures git_sha + sha256 of inputs/outputs
  + record_counts + validation_pass.
- reports/distillation/latest.md aggregating most-recent run of each stage.

Carry-overs to Phase 3 v2:
- mode_experiments scoring (168 needs_human_review): derive markers from
  validation_results.grounded_fraction
- extraction-class JOIN: distilled_*/audit_facts/observer_escalations
  → JOIN to verdict-bearing parent by task_id

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

auditor/schemas/distillation/preference_sample.ts

@@ -10,15 +10,14 @@ export const PREFERENCE_SAMPLE_SCHEMA_VERSION = 1;
 
 export interface PreferenceSample {
   schema_version: number;
+  id: string;
   prompt: string;
   chosen: string;
   rejected: string;
   reason: string;                  // why chosen > rejected — must be non-empty
-  source_run_ids: {
-    chosen: string;
-    rejected: string;
-  };
-  exported_at: string;
+  chosen_run_id: string;
+  rejected_run_id: string;
+  created_at: string;
   provenance: { source_file: string; line_offset?: number; sig_hash: string; recorded_at: string };
 }
 
@@ -32,11 +31,14 @@ export function validatePreferenceSample(input: unknown): ValidationResult<Prefe
     errors.push(`schema_version: expected ${PREFERENCE_SAMPLE_SCHEMA_VERSION}, got ${JSON.stringify(r.schema_version)}`);
     ok = false;
   }
+  ok = requireString(r.id, "id", errors) && ok;
   ok = requireString(r.prompt, "prompt", errors) && ok;
   ok = requireString(r.chosen, "chosen", errors) && ok;
   ok = requireString(r.rejected, "rejected", errors) && ok;
   ok = requireString(r.reason, "reason", errors) && ok;
-  ok = requireIsoTimestamp(r.exported_at, "exported_at", errors) && ok;
+  ok = requireString(r.chosen_run_id, "chosen_run_id", errors) && ok;
+  ok = requireString(r.rejected_run_id, "rejected_run_id", errors) && ok;
+  ok = requireIsoTimestamp(r.created_at, "created_at", errors) && ok;
   ok = requireProvenance(r.provenance, "provenance", errors) && ok;
 
   // Self-pairing guard.
@@ -44,22 +46,14 @@ export function validatePreferenceSample(input: unknown): ValidationResult<Prefe
     errors.push("chosen and rejected must differ — preference data needs a real disagreement");
     ok = false;
   }
+  if (r.chosen_run_id === r.rejected_run_id && typeof r.chosen_run_id === "string") {
+    errors.push("chosen_run_id and rejected_run_id must differ — same run can't disagree with itself");
+    ok = false;
+  }
   if (typeof r.reason === "string" && (r.reason as string).trim().length === 0) {
     errors.push("reason: must be non-whitespace (every preference needs WHY chosen > rejected)");
     ok = false;
   }
-  if (typeof r.source_run_ids !== "object" || r.source_run_ids === null) {
-    errors.push("source_run_ids: expected object {chosen, rejected}");
-    ok = false;
-  } else {
-    const s = r.source_run_ids as Record<string, unknown>;
-    ok = requireString(s.chosen, "source_run_ids.chosen", errors) && ok;
-    ok = requireString(s.rejected, "source_run_ids.rejected", errors) && ok;
-    if (s.chosen === s.rejected && typeof s.chosen === "string") {
-      errors.push("source_run_ids.chosen and .rejected must differ — same run can't disagree with itself");
-      ok = false;
-    }
-  }
 
   if (!ok) return { valid: false, errors };
   return { valid: true, value: r as unknown as PreferenceSample };

auditor/schemas/distillation/rag_sample.ts

@@ -6,6 +6,12 @@ import {
 
 export const RAG_SAMPLE_SCHEMA_VERSION = 1;
 
+// Allowed source_category values. RAG accepts accepted/partial freely;
+// needs_human_review is opt-in (must be tagged so consumers can filter
+// it out for SFT).
+export const RAG_ALLOWED_CATEGORIES = ["accepted", "partially_accepted", "needs_human_review"] as const;
+export type RagSourceCategory = (typeof RAG_ALLOWED_CATEGORIES)[number];
+
 export interface RagSample {
   schema_version: number;
   id: string;
@@ -13,9 +19,15 @@ export interface RagSample {
   content: string;
   tags: string[];
   source_run_id: string;
-  success_score: "accepted" | "partially_accepted";  // RAG only ships from these two
+  // Snapshot of the score the source carried at export time. Lets a
+  // consumer see "this was partial" without re-reading scored-runs.
+  success_score: RagSourceCategory;
+  // Same value as success_score by spec (now.md asks for both fields).
+  // Kept distinct so future schemas can diverge them (e.g. an
+  // "is_review_material" flag) without breaking old consumers.
+  source_category: RagSourceCategory;
   embedding_text: string;                            // the text to embed (often == content but can be shorter)
-  exported_at: string;
+  created_at: string;
   provenance: { source_file: string; line_offset?: number; sig_hash: string; recorded_at: string };
 }
 
@@ -34,12 +46,20 @@ export function validateRagSample(input: unknown): ValidationResult<RagSample> {
   ok = requireString(r.content, "content", errors) && ok;
   ok = requireString(r.embedding_text, "embedding_text", errors) && ok;
   ok = requireString(r.source_run_id, "source_run_id", errors) && ok;
-  ok = requireIsoTimestamp(r.exported_at, "exported_at", errors) && ok;
+  ok = requireIsoTimestamp(r.created_at, "created_at", errors) && ok;
   ok = requireStringArray(r.tags, "tags", errors) && ok;
   ok = requireProvenance(r.provenance, "provenance", errors) && ok;
 
-  if (!["accepted", "partially_accepted"].includes(r.success_score as string)) {
-    errors.push("success_score: must be accepted|partially_accepted (rejected/needs_human never enter RAG)");
+  if (!RAG_ALLOWED_CATEGORIES.includes(r.success_score as RagSourceCategory)) {
+    errors.push(`success_score: must be one of ${RAG_ALLOWED_CATEGORIES.join("|")} (rejected never enters RAG)`);
+    ok = false;
+  }
+  if (!RAG_ALLOWED_CATEGORIES.includes(r.source_category as RagSourceCategory)) {
+    errors.push(`source_category: must be one of ${RAG_ALLOWED_CATEGORIES.join("|")}`);
+    ok = false;
+  }
+  if (r.success_score !== r.source_category) {
+    errors.push("success_score and source_category must match (mirrored fields per spec)");
     ok = false;
   }
   if (typeof r.content === "string" && (r.content as string).trim().length === 0) {

auditor/schemas/distillation/schemas.test.ts

@@ -236,8 +236,9 @@ const RAG_OK = {
   tags: ["scrum_review", "applier"],
   source_run_id: "run-xyz",
   success_score: "accepted",
+  source_category: "accepted",
   embedding_text: "applier rationale-diff alignment guard scrum",
-  exported_at: NOW,
+  created_at: NOW,
   provenance: PROVENANCE,
 };
 
@@ -247,12 +248,17 @@ test("RagSample: positive validates", () => {
   expect(r.valid).toBe(true);
 });
 
-test("RagSample: success_score=rejected forbidden (RAG only takes accepted+partial)", () => {
-  const r = validateRagSample({ ...RAG_OK, success_score: "rejected" });
+test("RagSample: success_score=rejected forbidden (RAG never takes rejected)", () => {
+  const r = validateRagSample({ ...RAG_OK, success_score: "rejected", source_category: "rejected" });
   expect(r.valid).toBe(false);
   if (!r.valid) expect(r.errors.some(e => e.includes("success_score"))).toBe(true);
 });
 
+test("RagSample: success_score and source_category must match", () => {
+  const r = validateRagSample({ ...RAG_OK, success_score: "accepted", source_category: "partially_accepted" });
+  expect(r.valid).toBe(false);
+});
+
 test("RagSample: whitespace-only content rejected", () => {
   const r = validateRagSample({ ...RAG_OK, content: "   \n  " });
   expect(r.valid).toBe(false);
@@ -263,12 +269,13 @@ test("RagSample: whitespace-only content rejected", () => {
 
 const SFT_OK = {
   schema_version: SFT_SAMPLE_SCHEMA_VERSION,
+  id: "sft-pr11-001",
   instruction: "Audit this PR diff against ship-claims.",
   context: "claims: 3 strong, 2 moderate",
   response: "{\"claim_verdicts\": [...]}",
   source_run_id: "run-pr11",
   quality_score: "accepted",
-  exported_at: NOW,
+  created_at: NOW,
   provenance: PROVENANCE,
 };
 
@@ -278,15 +285,16 @@ test("SftSample: positive validates", () => {
   expect(r.valid).toBe(true);
 });
 
-test("SftSample: quality_score=partially_accepted REJECTED (spec non-negotiable, no leak)", () => {
+test("SftSample: quality_score=partially_accepted ACCEPTED (--include-partial path)", () => {
+  // Phase 4 update: partial allowed at schema layer; CLI gate decides.
   const r = validateSftSample({ ...SFT_OK, quality_score: "partially_accepted" });
-  expect(r.valid).toBe(false);
-  if (!r.valid) expect(r.errors.some(e => e.includes("quality_score"))).toBe(true);
+  expect(r.valid).toBe(true);
 });
 
 test("SftSample: quality_score=rejected REJECTED (spec non-negotiable, no leak)", () => {
   const r = validateSftSample({ ...SFT_OK, quality_score: "rejected" });
   expect(r.valid).toBe(false);
+  if (!r.valid) expect(r.errors.some(e => e.includes("quality_score"))).toBe(true);
 });
 
 test("SftSample: quality_score=needs_human_review REJECTED (no leak)", () => {
@@ -294,6 +302,19 @@ test("SftSample: quality_score=needs_human_review REJECTED (no leak)", () => {
   expect(r.valid).toBe(false);
 });
 
+test("SftSample: missing context rejected (must be string, even if empty)", () => {
+  const fixture: Record<string, unknown> = { ...SFT_OK };
+  delete fixture.context;
+  const r = validateSftSample(fixture);
+  expect(r.valid).toBe(false);
+  if (!r.valid) expect(r.errors.some(e => e.includes("context"))).toBe(true);
+});
+
+test("SftSample: empty-string context allowed", () => {
+  const r = validateSftSample({ ...SFT_OK, context: "" });
+  expect(r.valid).toBe(true);
+});
+
 test("SftSample: empty response rejected (no empty pairs)", () => {
   const r = validateSftSample({ ...SFT_OK, response: "" });
   expect(r.valid).toBe(false);
@@ -310,12 +331,14 @@ test("SftSample: whitespace-only instruction rejected", () => {
 
 const PREF_OK = {
   schema_version: PREFERENCE_SAMPLE_SCHEMA_VERSION,
+  id: "pref-task-x-001",
   prompt: "Verify claim: 'all 3 services running on matrix-test'",
   chosen: "{\"backed\": true, \"evidence\": \"systemctl status confirms 3 active\"}",
   rejected: "{\"backed\": true, \"evidence\": \"the README says so\"}",
   reason: "chosen cites runtime evidence, rejected cites doc claim only",
-  source_run_ids: { chosen: "run-A", rejected: "run-B" },
-  exported_at: NOW,
+  chosen_run_id: "run-A",
+  rejected_run_id: "run-B",
+  created_at: NOW,
   provenance: PROVENANCE,
 };
 
@@ -331,10 +354,10 @@ test("PreferenceSample: chosen == rejected rejected (no self-pairing)", () => {
   if (!r.valid) expect(r.errors.some(e => e.includes("chosen and rejected"))).toBe(true);
 });
 
-test("PreferenceSample: source_run_ids both equal rejected", () => {
-  const r = validatePreferenceSample({ ...PREF_OK, source_run_ids: { chosen: "run-A", rejected: "run-A" } });
+test("PreferenceSample: chosen_run_id == rejected_run_id rejected (no self-disagreement)", () => {
+  const r = validatePreferenceSample({ ...PREF_OK, chosen_run_id: "run-A", rejected_run_id: "run-A" });
   expect(r.valid).toBe(false);
-  if (!r.valid) expect(r.errors.some(e => e.includes("source_run_ids"))).toBe(true);
+  if (!r.valid) expect(r.errors.some(e => e.includes("chosen_run_id"))).toBe(true);
 });
 
 test("PreferenceSample: empty reason rejected (every preference needs WHY)", () => {

auditor/schemas/distillation/sft_sample.ts

@@ -7,14 +7,21 @@ import {
 
 export const SFT_SAMPLE_SCHEMA_VERSION = 1;
 
+// SFT default: only `accepted` ships. With --include-partial CLI flag,
+// `partially_accepted` becomes legal. `rejected` and `needs_human_review`
+// NEVER ship to SFT — that's the contamination firewall.
+export const SFT_QUALITY_SCORES = ["accepted", "partially_accepted"] as const;
+export type SftQualityScore = (typeof SFT_QUALITY_SCORES)[number];
+
 export interface SftSample {
   schema_version: number;
+  id: string;
   instruction: string;             // the prompt / user message
-  context?: string;                // optional retrieved context that was visible
+  context: string;                 // retrieved context that was visible (empty string allowed; null/undefined not)
   response: string;                // the model output that was accepted
   source_run_id: string;
-  quality_score: "accepted";       // hard-pinned — see validator
-  exported_at: string;
+  quality_score: SftQualityScore;
+  created_at: string;
   provenance: { source_file: string; line_offset?: number; sig_hash: string; recorded_at: string };
 }
 
@@ -28,10 +35,11 @@ export function validateSftSample(input: unknown): ValidationResult<SftSample> {
     errors.push(`schema_version: expected ${SFT_SAMPLE_SCHEMA_VERSION}, got ${JSON.stringify(r.schema_version)}`);
     ok = false;
   }
+  ok = requireString(r.id, "id", errors) && ok;
   ok = requireString(r.instruction, "instruction", errors) && ok;
   ok = requireString(r.response, "response", errors) && ok;
   ok = requireString(r.source_run_id, "source_run_id", errors) && ok;
-  ok = requireIsoTimestamp(r.exported_at, "exported_at", errors) && ok;
+  ok = requireIsoTimestamp(r.created_at, "created_at", errors) && ok;
   ok = requireProvenance(r.provenance, "provenance", errors) && ok;
 
   // Empty pair guard.
@@ -43,14 +51,16 @@ export function validateSftSample(input: unknown): ValidationResult<SftSample> {
     errors.push("response: must be non-whitespace (no empty pairs)");
     ok = false;
   }
-  // The non-negotiable: SFT samples MUST have quality_score=accepted.
-  // Anything else is a leak from rejected/partial/needs_human into SFT.
-  if (r.quality_score !== "accepted") {
-    errors.push(`quality_score: must be 'accepted' (no rejected/partial/needs_human leak into SFT — spec non-negotiable). Got ${JSON.stringify(r.quality_score)}`);
+  // Context is required-string but empty is allowed (some SFT samples
+  // are pure instruction→response with no retrieval context).
+  if (typeof r.context !== "string") {
+    errors.push("context: expected string (use empty string for no-context samples)");
     ok = false;
   }
-  if (r.context !== undefined && typeof r.context !== "string") {
-    errors.push("context: expected string when present");
+  // The non-negotiable: SFT samples MUST have quality_score in
+  // SFT_QUALITY_SCORES. Anything else is a leak.
+  if (!SFT_QUALITY_SCORES.includes(r.quality_score as SftQualityScore)) {
+    errors.push(`quality_score: must be one of ${SFT_QUALITY_SCORES.join("|")} (no rejected/needs_human leak into SFT — spec non-negotiable). Got ${JSON.stringify(r.quality_score)}`);
     ok = false;
   }