catalogd: HTML-safe escape fix + decisions tracker entry
Per 2026-05-03 step_7_8_retention_and_parity scrum (opus WARN on
parity_subject_audit.rs:canonical_json):
Go's json.Marshal HTML-escapes < > & to < > & by
default. Rust's serde_json::to_vec keeps them literal. Any audit
row with these chars in any string field would silently produce
different canonical bytes across runtimes → broken HMAC chain.
Latent because no production audit field has carried <>& yet, but
realistic for purpose strings ("error & retry") or trace_id values
("<HTTP-Request-Id>").
Fix: marshalNoEscapeHTML helper wraps json.Encoder.SetEscapeHTML(false)
+ trims trailing newline. Routed through writeCanonical for both
keys and scalar values.
Regression test: TestVerifyChain_HtmlChars_NotEscaped (purpose has &,
trace_id has <>) asserts the canonical bytes contain literal chars,
not escape sequences.
11 unit tests pass including the new one; parity probe still 6/6
byte-identical against live production audit logs.
Decisions tracker: added 2026-05-03 entry for SUBJECT_MANIFESTS_ON_CATALOGD
Steps 1-8 closure + 6th cross-runtime parity probe (was 5).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,7 @@
|
||||
package catalogd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
@@ -203,9 +204,36 @@ func canonicalRowBytesFromStruct(row *SubjectAuditRow) ([]byte, error) {
|
||||
return canonicalRowBytesFromRaw(raw)
|
||||
}
|
||||
|
||||
// marshalNoEscapeHTML wraps json.Encoder with HTML escaping disabled.
|
||||
//
|
||||
// Why: Go's json.Marshal escapes `<`, `>`, `&` to `<`, `>`,
|
||||
// `&` by default. Rust's serde_json::to_vec keeps them literal.
|
||||
// Any string field containing one of those characters would produce
|
||||
// different canonical bytes across runtimes → broken HMAC chain.
|
||||
// (Caught 2026-05-03 by opus scrum WARN on parity_subject_audit.rs:
|
||||
// canonical_json — initially undetected because no production audit
|
||||
// field contained `<>&`, but realistic for purpose strings like
|
||||
// "error & retry" or trace_id "<HTTP-Request-Id>".)
|
||||
//
|
||||
// Also strips the trailing newline json.Encoder appends — that newline
|
||||
// is meaningful to JSONL consumers but is junk for hash input.
|
||||
func marshalNoEscapeHTML(v any) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := buf.Bytes()
|
||||
return bytes.TrimRight(out, "\n"), nil
|
||||
}
|
||||
|
||||
// writeCanonical recursively writes v as canonical JSON: object keys
|
||||
// sorted alphabetically, no insignificant whitespace. Arrays preserve
|
||||
// element order (semantically significant per spec §3).
|
||||
//
|
||||
// All scalar emission goes through marshalNoEscapeHTML so the byte
|
||||
// sequence matches Rust's serde_json output character-for-character.
|
||||
func writeCanonical(buf *strings.Builder, v any) error {
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
@@ -219,7 +247,7 @@ func writeCanonical(buf *strings.Builder, v any) error {
|
||||
if i > 0 {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
ks, err := json.Marshal(k)
|
||||
ks, err := marshalNoEscapeHTML(k)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
@@ -242,9 +270,7 @@ func writeCanonical(buf *strings.Builder, v any) error {
|
||||
}
|
||||
buf.WriteByte(']')
|
||||
default:
|
||||
// json.Number, string, bool, nil — encoding/json renders these
|
||||
// the same way Rust's serde_json does (compact, RFC-8259-conformant).
|
||||
bs, err := json.Marshal(v)
|
||||
bs, err := marshalNoEscapeHTML(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal scalar: %w", err)
|
||||
}
|
||||
|
||||
@@ -237,6 +237,37 @@ func TestKnownAnswerVector(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifyChain_HtmlChars_NotEscaped is the regression test for the
|
||||
// 2026-05-03 opus scrum WARN: Go's json.Marshal escapes `<`, `>`, `&`
|
||||
// to `<`, `>`, `&` by default; Rust's serde_json keeps
|
||||
// them literal. Audit rows with these chars in any string field would
|
||||
// silently break the chain across runtimes. Fix is in writeCanonical's
|
||||
// marshalNoEscapeHTML helper. This test asserts canonical bytes contain
|
||||
// the literal `<`, `>`, `&` (proving the fix is in place).
|
||||
func TestVerifyChain_HtmlChars_NotEscaped(t *testing.T) {
|
||||
r := mkRow("CAND-HTML", []string{"name"}, GenesisHash, "2026-05-03T12:00:00Z")
|
||||
r.Accessor.Purpose = "error & retry" // & must NOT be &
|
||||
r.Accessor.TraceID = "<HTTP-Req-Id>" // < and > must NOT be < / >
|
||||
canon, err := canonicalRowBytesFromStruct(&r)
|
||||
if err != nil {
|
||||
t.Fatalf("canonical: %v", err)
|
||||
}
|
||||
s := string(canon)
|
||||
// FAIL if the bytes contain Go's HTML-safe < / > / &
|
||||
// escape sequences (six raw chars each: backslash, u, 0, 0, hex, hex).
|
||||
// Those wouldn't match Rust's literal-char output and would silently
|
||||
// break the cross-runtime HMAC chain. Note: the strings below are
|
||||
// raw-string literals — the backslash + u006xx is six literal bytes,
|
||||
// NOT a Go-source unicode escape.
|
||||
if strings.Contains(s, "\\u003c") || strings.Contains(s, "\\u003e") || strings.Contains(s, "\\u0026") {
|
||||
t.Fatalf("canonical bytes contain Go HTML-escape sequences (would diverge from Rust):\n%s", s)
|
||||
}
|
||||
// PASS only if the literal chars survived round-trip.
|
||||
if !strings.Contains(s, "\"<HTTP-Req-Id>\"") || !strings.Contains(s, "\"error & retry\"") {
|
||||
t.Fatalf("canonical bytes missing literal <>&:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifyChain_RawBytesPreserveTimePrecision is the regression test
|
||||
// for the 2026-05-03 WORKER-5 finding: when a row's nanoseconds end in
|
||||
// 0, time.RFC3339Nano strips the trailing zero on re-marshal, producing
|
||||
|
||||
Reference in New Issue
Block a user