G0 D5: queryd DuckDB SELECT over Parquet via httpfs · 4 scrum fixes

Phase G0 Day 5 ships queryd: in-memory DuckDB with custom Connector
that runs INSTALL httpfs / LOAD httpfs / CREATE OR REPLACE SECRET
(TYPE S3) on every new connection, sourced from SecretsProvider +
shared.S3Config. SetMaxOpenConns(1) so registrar's CREATE VIEWs and
handler's SELECTs serialize through one connection (avoids cross-
connection MVCC visibility edge cases).

Registrar.Refresh reads catalogd /catalog/list, runs CREATE OR
REPLACE VIEW "name" AS SELECT * FROM read_parquet('s3://bucket/key')
per manifest, drops views for removed manifests, skips on unchanged
updated_at (the implicit etag). Drop pass runs BEFORE create pass so
a poison manifest can't block other manifest refreshes (post-scrum
C1 fix).

POST /sql with JSON body {"sql":"…"} returns
{"columns":[{"name":"id","type":"BIGINT"},…], "rows":[[…]],
"row_count":N}. []byte → string conversion so VARCHAR rows
JSON-encode as text. 30s default refresh ticker, configurable via
[queryd].refresh_every.

Cross-lineage scrum on shipped code:
  - Opus 4.7 (opencode):                      1 BLOCK + 4 WARN + 4 INFO
  - Kimi K2-0905 (openrouter):                2 BLOCK + 2 WARN + 1 INFO
  - Qwen3-coder (openrouter):                 2 BLOCK + 1 WARN + 1 INFO

Fixed (4):
  C1 (Opus + Kimi convergent): Refresh aborts on first per-view error
    → drop pass first, collect errors, errors.Join. Poison manifest
    no longer blocks the rest of the catalog from re-syncing.
  B-CTX (Opus BLOCK): bootstrap closure captured OpenDB's ctx →
    cancelled-ctx silently fails every reconnect. context.Background()
    inside closure; passed ctx only for initial Ping.
  B-LEAK (Kimi BLOCK): firstLine(stmt) truncated CREATE SECRET to 80
    chars but those 80 chars contained KEY_ID + SECRET prefix → log
    aggregator captures credentials. Stable per-statement labels +
    redactCreds() filter on wrapped DuckDB errors.
  JSON-ERR (Opus WARN): swallowed json.Encode error → silent
    truncated 200 on unsupported column types. slog.Warn the failure.

Dismissed (4 false positives):
  Qwen BLOCK "bootstrap not transactional" — DuckDB DDL is auto-commit
  Qwen BLOCK "MaxBytesReader after Decode" — false, applied before
  Kimi BLOCK "concurrent Refresh + user SELECT deadlock" — not a
    deadlock, just serialization, by design with 10s timeout retry
  Kimi WARN "dropView leaves r.known inconsistent" — current code
    returns before the delete; the entry persists for retry

Critical reviewer behavior: 1 convergent BLOCK between Opus + Kimi
on the per-view error blocking, plus two independent single-reviewer
BLOCKs (B-CTX, B-LEAK) that smoke could never have caught. The
B-LEAK fix uses defense-in-depth: never pass SQL into the error
path AND redact known cred values from DuckDB's own error message.

DuckDB cgo path: github.com/duckdb/duckdb-go/v2 v2.10502.0 (per
ADR-001 §1) on Go 1.25 + arrow-go. Smoke 6/6 PASS after every
fix round.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

internal/queryd/db.go

@@ -0,0 +1,140 @@
+// Package queryd is the SQL execution surface — a DuckDB engine that
+// reads Parquet directly from S3-compatible storage (MinIO in dev)
+// via DuckDB's httpfs extension. Views are registered from catalogd
+// manifests so user SQL just references dataset names.
+//
+// db.go owns the *sql.DB lifecycle: a custom Connector with a
+// bootstrapper that runs INSTALL httpfs / LOAD httpfs / CREATE OR
+// REPLACE SECRET on every new connection, plus a SetMaxOpenConns(1)
+// pin so the registrar's CREATE VIEWs and the handler's user SQL
+// serialize through one connection (avoids cross-connection visibility
+// edge cases for G0; lift to a pool when concurrency wins matter).
+package queryd
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+	"strings"
+
+	"github.com/duckdb/duckdb-go/v2"
+
+	"git.agentview.dev/profit/golangLAKEHOUSE/internal/secrets"
+	"git.agentview.dev/profit/golangLAKEHOUSE/internal/shared"
+)
+
+// OpenDB returns a *sql.DB backed by an in-memory DuckDB whose
+// connections are pre-loaded with httpfs + an S3 secret derived from
+// the shared S3 config + secrets provider. The bucket parameter names
+// the logical bucket whose credentials are pulled.
+//
+// Caller is responsible for Close()ing the returned db.
+func OpenDB(ctx context.Context, s3 shared.S3Config, prov secrets.Provider, bucketLogicalName string) (*sql.DB, error) {
+	creds, err := prov.S3Credentials(bucketLogicalName)
+	if err != nil {
+		return nil, fmt.Errorf("queryd: secrets: %w", err)
+	}
+
+	bootstrap := buildBootstrap(s3, creds)
+	// Stable labels per bootstrap statement. Per scrum B-LEAK (Kimi):
+	// the prior firstLine(stmt) truncated CREATE OR REPLACE SECRET
+	// to 80 chars, which contained both KEY_ID and the start of
+	// SECRET — a log aggregator would capture credentials. Using
+	// stable labels avoids putting the SQL into the error path at all.
+	labels := []string{"install httpfs", "load httpfs", "create secret"}
+
+	connector, err := duckdb.NewConnector("", func(execer driver.ExecerContext) error {
+		// Per scrum B-CTX (Opus): use Background() inside the bootstrap
+		// closure rather than capturing the OpenDB-call ctx. The
+		// connector callback runs on EVERY new physical connection,
+		// including reconnects long after OpenDB returned. A captured
+		// short-lived ctx would silently fail every reconnect's
+		// bootstrap. The passed ctx is only for the initial Ping below.
+		for i, stmt := range bootstrap {
+			if _, err := execer.ExecContext(context.Background(), stmt, nil); err != nil {
+				return fmt.Errorf("queryd bootstrap %s: %s", labels[i], redactCreds(err.Error(), creds))
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("queryd: new connector: %w", err)
+	}
+
+	db := sql.OpenDB(connector)
+	// One connection: the registrar's view CREATEs are visible to the
+	// query handler's SELECTs without worrying about MVCC/timing.
+	// Lift this for G2+ when concurrent reads matter.
+	db.SetMaxOpenConns(1)
+	if err := db.PingContext(ctx); err != nil {
+		_ = db.Close()
+		return nil, fmt.Errorf("queryd: ping: %w", err)
+	}
+	return db, nil
+}
+
+// buildBootstrap returns the SQL statements that initialize a fresh
+// DuckDB connection: install + load httpfs, then create-or-replace
+// the unnamed S3 secret. Endpoint is normalized — DuckDB wants
+// `host:port` without the scheme; the http/https split is controlled
+// by USE_SSL.
+func buildBootstrap(s3 shared.S3Config, creds secrets.S3Credentials) []string {
+	endpoint := s3.Endpoint
+	useSSL := true
+	switch {
+	case strings.HasPrefix(endpoint, "http://"):
+		endpoint = strings.TrimPrefix(endpoint, "http://")
+		useSSL = false
+	case strings.HasPrefix(endpoint, "https://"):
+		endpoint = strings.TrimPrefix(endpoint, "https://")
+		useSSL = true
+	}
+	urlStyle := "vhost"
+	if s3.UsePathStyle {
+		urlStyle = "path"
+	}
+
+	// CREATE OR REPLACE so a reconnect doesn't error on the existing
+	// secret. Single-quoted SQL string literals; we escape ' → '' for
+	// belt-and-braces (creds shouldn't contain ' but a future SSO
+	// token might).
+	createSecret := fmt.Sprintf(
+		"CREATE OR REPLACE SECRET (TYPE S3, KEY_ID '%s', SECRET '%s', REGION '%s', ENDPOINT '%s', URL_STYLE '%s', USE_SSL %t)",
+		sqlEscape(creds.AccessKeyID),
+		sqlEscape(creds.SecretAccessKey),
+		sqlEscape(s3.Region),
+		sqlEscape(endpoint),
+		sqlEscape(urlStyle),
+		useSSL,
+	)
+
+	return []string{
+		"INSTALL httpfs",
+		"LOAD httpfs",
+		createSecret,
+	}
+}
+
+// sqlEscape doubles single quotes in a SQL string literal value.
+// SQL identifier escaping (doubling " for quoted identifiers) lives
+// next to its use site in registrar.go.
+func sqlEscape(v string) string {
+	return strings.ReplaceAll(v, "'", "''")
+}
+
+// redactCreds replaces known credential values inside an error
+// message with placeholder tokens. DuckDB's parser/exec error may
+// echo the offending statement; this scrubs the secret values
+// regardless. Per scrum B-LEAK (Kimi): a 503 from a typo in CREATE
+// SECRET would otherwise drop both KEY_ID and SECRET into the log
+// pipeline.
+func redactCreds(msg string, creds secrets.S3Credentials) string {
+	if creds.AccessKeyID != "" {
+		msg = strings.ReplaceAll(msg, creds.AccessKeyID, "[REDACTED-KEY]")
+	}
+	if creds.SecretAccessKey != "" {
+		msg = strings.ReplaceAll(msg, creds.SecretAccessKey, "[REDACTED-SECRET]")
+	}
+	return msg
+}