proof harness Phase A: scaffolding + canary case green

Per docs/TEST_PROOF_SCOPE.md, building the claims-verification tier above the smoke chain. This commit lays the scaffolding and proves the orchestrator end-to-end with one canary case (00_health). What landed: tests/proof/ README.md how to read a report, layout, modes claims.yaml 24 claims enumerated (GOLAKE-001..100) run_proof.sh orchestrator with --mode {contract|integration|performance} and --no-bootstrap / --regenerate-{rankings,baseline} lib/ env.sh service URLs, report dir, mode, git context http.sh curl wrappers writing per-probe JSON + body + headers assert.sh proof_assert_{eq,ne,contains,lt,gt,status,json_eq} + proof_skip — each emits one JSONL record per call metrics.sh start/stop timers, value capture, RSS sampling, percentile compute (for Phase D) cases/ 00_health.sh canary — gateway + 6 services /health → 200, body identifies service, latency < 500ms (21 assertions) fixtures/ csv/workers.csv spec's 5-row deterministic CSV text/docs.txt 4 deterministic vector docs expected/queries.json expected results for the 5 SQL assertions Wired into the task runner: just proof contract # canary only this commit just proof integration # Phase C just proof performance # Phase D .gitignore: /tests/proof/reports/* with !.gitkeep — same pattern as reports/scrum/_evidence/. Per-run output is a runtime artifact. Specs landed alongside (J's drops): docs/TEST_PROOF_SCOPE.md the harness contract this implements docs/CLAUDE_REFACTOR_GUARDRAILS.md process discipline this harness obeys Verified end-to-end (cached binaries): just proof contract wall < 2s, 21 pass / 0 fail / 0 skip just verify wall 31s, vet + test + 9 smokes still green Two bugs fixed during canary run, both in run_proof.sh aggregation: - grep -c exits 1 on zero matches; the `|| echo 0` form concatenated "0\n0" and broke jq --argjson + integer comparison. Fixed via a _count helper that captures count-or-zero cleanly. - per-case table iterated case scripts (filename-based) but cases write evidence under CASE_ID. Switched to JSONL-file iteration so multi-case scripts work and the mapping is faithful. Phase B (contract cases) lands next: 05_embedding, 06_vector_add, 08_gateway_contracts, 09_failure_modes. Each sourcing the same lib helpers and writing to the same report shape. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-29 05:08:51 -05:00
parent e31638204d
commit a81291e38c
16 changed files with 1582 additions and 0 deletions
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# lib/assert.sh — assertions that record evidence per the spec.
+#
+# Each assertion appends one record to:
+#   $PROOF_REPORT_DIR/raw/cases/<case_id>.jsonl
+#
+# Each line is a self-describing JSON object — case ID, claim, expected,
+# actual, result {pass|fail|skip}, optional evidence pointers. Cases
+# call multiple assertions; run_proof.sh aggregates JSONL → summary.
+#
+# Functions:
+#   proof_assert_eq        <case_id> <claim> <expected> <actual>
+#   proof_assert_ne        <case_id> <claim> <not_expected> <actual>
+#   proof_assert_contains  <case_id> <claim> <substring> <haystack>
+#   proof_assert_lt        <case_id> <claim> <a> <b>          # passes if a < b
+#   proof_assert_gt        <case_id> <claim> <a> <b>          # passes if a > b
+#   proof_assert_status    <case_id> <claim> <expected_status> <probe_name>
+#   proof_assert_json_eq   <case_id> <claim> <jq_path> <expected> <body_or_file>
+#   proof_skip             <case_id> <claim> <reason>
+#
+# All return 0 (case scripts decide their own halt-on-fail policy).
+
+_proof_record() {
+    local case_id="$1" claim="$2" result="$3" expected="$4" actual="$5" detail="$6"
+    local out="${PROOF_REPORT_DIR}/raw/cases/${case_id}.jsonl"
+    mkdir -p "$(dirname "$out")"
+    # JSON-escape the variable inputs.
+    local e_claim e_expected e_actual e_detail
+    e_claim=$(printf '%s' "$claim"       | jq -Rs .)
+    e_expected=$(printf '%s' "$expected" | jq -Rs .)
+    e_actual=$(printf '%s' "$actual"     | jq -Rs .)
+    e_detail=$(printf '%s' "$detail"     | jq -Rs .)
+    local ts
+    ts=$(date -u -Iseconds)
+    cat >> "$out" <<JSON
+{"case_id":"${case_id}","claim":${e_claim},"result":"${result}","expected":${e_expected},"actual":${e_actual},"detail":${e_detail},"timestamp":"${ts}","git_sha":"${PROOF_GIT_SHA}"}
+JSON
+}
+
+proof_assert_eq() {
+    local case_id="$1" claim="$2" expected="$3" actual="$4"
+    if [ "$expected" = "$actual" ]; then
+        _proof_record "$case_id" "$claim" pass "$expected" "$actual" ""
+        return 0
+    fi
+    _proof_record "$case_id" "$claim" fail "$expected" "$actual" "values differ"
+    return 0
+}
+
+proof_assert_ne() {
+    local case_id="$1" claim="$2" not_expected="$3" actual="$4"
+    if [ "$not_expected" != "$actual" ]; then
+        _proof_record "$case_id" "$claim" pass "!= ${not_expected}" "$actual" ""
+        return 0
+    fi
+    _proof_record "$case_id" "$claim" fail "!= ${not_expected}" "$actual" "values matched (should differ)"
+    return 0
+}
+
+proof_assert_contains() {
+    local case_id="$1" claim="$2" substring="$3" haystack="$4"
+    if [[ "$haystack" == *"$substring"* ]]; then
+        _proof_record "$case_id" "$claim" pass "contains: ${substring}" "$haystack" ""
+        return 0
+    fi
+    _proof_record "$case_id" "$claim" fail "contains: ${substring}" "$haystack" "substring not found"
+    return 0
+}
+
+proof_assert_lt() {
+    local case_id="$1" claim="$2" a="$3" b="$4"
+    # awk handles ints + floats uniformly
+    if awk -v a="$a" -v b="$b" 'BEGIN{exit !(a < b)}'; then
+        _proof_record "$case_id" "$claim" pass "${a} < ${b}" "${a}" ""
+        return 0
+    fi
+    _proof_record "$case_id" "$claim" fail "${a} < ${b}" "${a}" "${a} is not less than ${b}"
+    return 0
+}
+
+proof_assert_gt() {
+    local case_id="$1" claim="$2" a="$3" b="$4"
+    if awk -v a="$a" -v b="$b" 'BEGIN{exit !(a > b)}'; then
+        _proof_record "$case_id" "$claim" pass "${a} > ${b}" "${a}" ""
+        return 0
+    fi
+    _proof_record "$case_id" "$claim" fail "${a} > ${b}" "${a}" "${a} is not greater than ${b}"
+    return 0
+}
+
+# proof_assert_status compares the status from a previously-recorded
+# probe against an expected value. Probe must have run via lib/http.sh.
+proof_assert_status() {
+    local case_id="$1" claim="$2" expected="$3" probe_name="$4"
+    local actual
+    actual=$(proof_status_of "$case_id" "$probe_name" 2>/dev/null || echo missing)
+    proof_assert_eq "$case_id" "$claim" "$expected" "$actual"
+}
+
+# proof_assert_json_eq: jq-based equality on response body or a file.
+# body_or_file: if starts with @, read from file; otherwise treat as
+# literal JSON string.
+proof_assert_json_eq() {
+    local case_id="$1" claim="$2" jq_path="$3" expected="$4" source="$5"
+    local actual
+    if [[ "$source" == @* ]]; then
+        actual=$(jq -r "$jq_path" "${source#@}" 2>/dev/null || echo "<jq error>")
+    else
+        actual=$(printf '%s' "$source" | jq -r "$jq_path" 2>/dev/null || echo "<jq error>")
+    fi
+    proof_assert_eq "$case_id" "$claim" "$expected" "$actual"
+}
+
+proof_skip() {
+    local case_id="$1" claim="$2" reason="$3"
+    _proof_record "$case_id" "$claim" skip "" "" "$reason"
+    return 0
+}
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# lib/env.sh — proof harness environment.
+#
+# Sourced once by run_proof.sh and by every case script. Establishes:
+#   - service URLs (gateway and direct ports)
+#   - report directory paths
+#   - run context (git SHA, hostname, timestamp)
+#   - mode (contract|integration|performance)
+#
+# Cases read from these vars; never re-set them.
+
+# Repo root — every path the harness emits is anchored here so report
+# JSON is portable across reviewer machines.
+PROOF_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+export PROOF_REPO_ROOT
+
+# Service endpoints. Match internal/shared/config.go defaults; every
+# binary binds 127.0.0.1 per the audit's R-007 mitigation.
+export PROOF_GATEWAY_URL="${PROOF_GATEWAY_URL:-http://127.0.0.1:3110}"
+export PROOF_STORAGED_URL="${PROOF_STORAGED_URL:-http://127.0.0.1:3211}"
+export PROOF_CATALOGD_URL="${PROOF_CATALOGD_URL:-http://127.0.0.1:3212}"
+export PROOF_INGESTD_URL="${PROOF_INGESTD_URL:-http://127.0.0.1:3213}"
+export PROOF_QUERYD_URL="${PROOF_QUERYD_URL:-http://127.0.0.1:3214}"
+export PROOF_VECTORD_URL="${PROOF_VECTORD_URL:-http://127.0.0.1:3215}"
+export PROOF_EMBEDD_URL="${PROOF_EMBEDD_URL:-http://127.0.0.1:3216}"
+
+# Mode + report directory — set by run_proof.sh before sourcing cases.
+# Defaulted here so cases sourced standalone for debug still work.
+export PROOF_MODE="${PROOF_MODE:-contract}"
+
+if [ -z "${PROOF_REPORT_DIR:-}" ]; then
+    ts="$(date -u +%Y%m%d-%H%M%SZ)"
+    export PROOF_REPORT_DIR="${PROOF_REPO_ROOT}/tests/proof/reports/proof-${ts}"
+fi
+
+mkdir -p \
+    "${PROOF_REPORT_DIR}/raw/http" \
+    "${PROOF_REPORT_DIR}/raw/logs" \
+    "${PROOF_REPORT_DIR}/raw/outputs" \
+    "${PROOF_REPORT_DIR}/raw/metrics" \
+    "${PROOF_REPORT_DIR}/raw/cases"
+
+# Run context — captured once per run by run_proof.sh, but recomputed
+# here as fallback if a case is invoked standalone.
+if [ ! -f "${PROOF_REPORT_DIR}/raw/context.json" ]; then
+    git_sha="$(cd "$PROOF_REPO_ROOT" && git rev-parse HEAD 2>/dev/null || echo unknown)"
+    git_dirty="$(cd "$PROOF_REPO_ROOT" && [ -n "$(git status --porcelain 2>/dev/null)" ] && echo true || echo false)"
+    cat > "${PROOF_REPORT_DIR}/raw/context.json" <<JSON
+{
+  "git_sha": "${git_sha}",
+  "git_dirty": ${git_dirty},
+  "hostname": "$(hostname)",
+  "timestamp_utc": "$(date -u -Iseconds)",
+  "mode": "${PROOF_MODE}",
+  "harness_version": "1"
+}
+JSON
+fi
+
+export PROOF_GIT_SHA="$(cd "$PROOF_REPO_ROOT" && git rev-parse HEAD 2>/dev/null || echo unknown)"
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# lib/http.sh — curl wrappers that capture latency, status, body.
+#
+# Each request emits a JSON file under raw/http/<case_id>/<probe>.json
+# describing the round-trip. Cases consume the JSON via assert.sh.
+#
+# Why JSON files instead of bash variables: gives the final report a
+# diffable, replayable record. Future runs can compare on disk without
+# re-executing the case.
+#
+# Functions:
+#   proof_get   <case_id> <probe_name> <url> [extra-curl-args...]
+#   proof_post  <case_id> <probe_name> <url> <content-type> <body> [extra-curl-args...]
+#   proof_put   <case_id> <probe_name> <url> <content-type> <body|@file> [extra-curl-args...]
+#   proof_delete<case_id> <probe_name> <url> [extra-curl-args...]
+#
+# Returns 0 always (capture is independent of HTTP outcome).
+# Stores result at:  $PROOF_REPORT_DIR/raw/http/<case_id>/<probe>.json
+# Stores body at:    $PROOF_REPORT_DIR/raw/http/<case_id>/<probe>.body
+
+_proof_http_emit() {
+    local case_id="$1" probe="$2" method="$3" url="$4" status="$5" latency_ms="$6" body_path="$7" headers_path="$8"
+    local dir="${PROOF_REPORT_DIR}/raw/http/${case_id}"
+    mkdir -p "$dir"
+    local body_sha=""
+    [ -s "$body_path" ] && body_sha="$(sha256sum "$body_path" | awk '{print $1}')"
+    cat > "${dir}/${probe}.json" <<JSON
+{
+  "case_id": "${case_id}",
+  "probe": "${probe}",
+  "method": "${method}",
+  "url": "${url}",
+  "status": ${status},
+  "latency_ms": ${latency_ms},
+  "body_path": "raw/http/${case_id}/${probe}.body",
+  "body_sha256": "${body_sha}",
+  "headers_path": "raw/http/${case_id}/${probe}.headers"
+}
+JSON
+}
+
+# Internal common runner — populates a temp body+headers file, times
+# the request, emits the per-probe JSON, prints the body to stdout for
+# convenience (cases can capture or discard).
+_proof_http_run() {
+    local case_id="$1" probe="$2" method="$3" url="$4"; shift 4
+    local dir="${PROOF_REPORT_DIR}/raw/http/${case_id}"
+    mkdir -p "$dir"
+    local body_path="${dir}/${probe}.body"
+    local headers_path="${dir}/${probe}.headers"
+    local start_ms end_ms
+    start_ms=$(date +%s%3N)
+    local status
+    status=$(curl -sS -X "$method" -o "$body_path" -D "$headers_path" -w "%{http_code}" "$@" "$url" 2>/dev/null || echo 0)
+    end_ms=$(date +%s%3N)
+    local latency_ms=$((end_ms - start_ms))
+    _proof_http_emit "$case_id" "$probe" "$method" "$url" "$status" "$latency_ms" "$body_path" "$headers_path"
+    cat "$body_path"
+}
+
+proof_get() {
+    local case_id="$1" probe="$2" url="$3"; shift 3
+    _proof_http_run "$case_id" "$probe" GET "$url" "$@"
+}
+
+proof_post() {
+    local case_id="$1" probe="$2" url="$3" content_type="$4" body="$5"; shift 5
+    _proof_http_run "$case_id" "$probe" POST "$url" \
+        -H "Content-Type: ${content_type}" \
+        --data "$body" \
+        "$@"
+}
+
+# proof_put accepts either an inline body or @-prefixed file path
+# (curl --upload-file semantics for streaming).
+proof_put() {
+    local case_id="$1" probe="$2" url="$3" content_type="$4" body="$5"; shift 5
+    if [[ "$body" == @* ]]; then
+        local file="${body#@}"
+        _proof_http_run "$case_id" "$probe" PUT "$url" \
+            -H "Content-Type: ${content_type}" \
+            --upload-file "$file" \
+            "$@"
+    else
+        _proof_http_run "$case_id" "$probe" PUT "$url" \
+            -H "Content-Type: ${content_type}" \
+            --data "$body" \
+            "$@"
+    fi
+}
+
+proof_delete() {
+    local case_id="$1" probe="$2" url="$3"; shift 3
+    _proof_http_run "$case_id" "$probe" DELETE "$url" "$@"
+}
+
+# Helper accessors — reads the per-probe JSON.
+proof_status_of() {
+    local case_id="$1" probe="$2"
+    jq -r '.status' "${PROOF_REPORT_DIR}/raw/http/${case_id}/${probe}.json"
+}
+
+proof_body_of() {
+    local case_id="$1" probe="$2"
+    cat "${PROOF_REPORT_DIR}/raw/http/${case_id}/${probe}.body"
+}
+
+proof_latency_of() {
+    local case_id="$1" probe="$2"
+    jq -r '.latency_ms' "${PROOF_REPORT_DIR}/raw/http/${case_id}/${probe}.json"
+}
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# lib/metrics.sh — performance measurements for performance mode.
+#
+# Functions:
+#   proof_metric_start <case_id> <metric_name>
+#   proof_metric_stop  <case_id> <metric_name>           # writes elapsed_ms
+#   proof_metric_value <case_id> <metric_name> <value> [unit]
+#   proof_sample_rss   <case_id> <process_pattern>       # MB
+#   proof_compute_percentile <values_file> <percentile>  # 50, 95, 99
+#
+# All metrics emit to:
+#   $PROOF_REPORT_DIR/raw/metrics/<case_id>.jsonl
+
+_proof_metric_emit() {
+    local case_id="$1" name="$2" value="$3" unit="$4" detail="$5"
+    local out="${PROOF_REPORT_DIR}/raw/metrics/${case_id}.jsonl"
+    mkdir -p "$(dirname "$out")"
+    local e_detail
+    e_detail=$(printf '%s' "$detail" | jq -Rs .)
+    cat >> "$out" <<JSON
+{"case_id":"${case_id}","metric":"${name}","value":${value},"unit":"${unit}","detail":${e_detail},"timestamp":"$(date -u -Iseconds)"}
+JSON
+}
+
+proof_metric_start() {
+    local case_id="$1" name="$2"
+    local f="${PROOF_REPORT_DIR}/raw/metrics/_timer_${case_id}_${name}"
+    date +%s%3N > "$f"
+}
+
+proof_metric_stop() {
+    local case_id="$1" name="$2"
+    local f="${PROOF_REPORT_DIR}/raw/metrics/_timer_${case_id}_${name}"
+    if [ ! -f "$f" ]; then
+        echo "[metrics] timer ${name} for case ${case_id} not started" >&2
+        return 1
+    fi
+    local start_ms end_ms elapsed_ms
+    start_ms=$(cat "$f")
+    end_ms=$(date +%s%3N)
+    elapsed_ms=$((end_ms - start_ms))
+    rm -f "$f"
+    _proof_metric_emit "$case_id" "${name}_ms" "$elapsed_ms" "ms" ""
+    echo "$elapsed_ms"
+}
+
+proof_metric_value() {
+    local case_id="$1" name="$2" value="$3" unit="${4:-}"
+    _proof_metric_emit "$case_id" "$name" "$value" "$unit" ""
+}
+
+# Sample resident-set-size (MB) for the first matching process.
+proof_sample_rss() {
+    local case_id="$1" pattern="$2"
+    local pid rss_kb rss_mb
+    pid=$(pgrep -f "$pattern" | head -1)
+    if [ -z "$pid" ]; then
+        _proof_metric_emit "$case_id" "rss_${pattern//\//_}_mb" 0 "MB" "process not found"
+        return 1
+    fi
+    rss_kb=$(awk '/^VmRSS:/ {print $2}' "/proc/$pid/status" 2>/dev/null || echo 0)
+    rss_mb=$(awk -v k="$rss_kb" 'BEGIN{printf "%.1f", k/1024}')
+    _proof_metric_emit "$case_id" "rss_${pattern//\//_}_mb" "$rss_mb" "MB" "pid=${pid}"
+    echo "$rss_mb"
+}
+
+# proof_compute_percentile: streams values from a file (one number per
+# line), prints the requested percentile.
+proof_compute_percentile() {
+    local file="$1" pct="$2"
+    sort -n "$file" | awk -v pct="$pct" '
+        { v[NR] = $1 }
+        END {
+            n = NR
+            if (n == 0) { print "0"; exit }
+            idx = int((pct/100.0) * n)
+            if (idx < 1) idx = 1
+            if (idx > n) idx = n
+            print v[idx]
+        }
+    '
+}