G0 D5: queryd DuckDB SELECT over Parquet via httpfs · 4 scrum fixes
Phase G0 Day 5 ships queryd: in-memory DuckDB with custom Connector
that runs INSTALL httpfs / LOAD httpfs / CREATE OR REPLACE SECRET
(TYPE S3) on every new connection, sourced from SecretsProvider +
shared.S3Config. SetMaxOpenConns(1) so registrar's CREATE VIEWs and
handler's SELECTs serialize through one connection (avoids cross-
connection MVCC visibility edge cases).
Registrar.Refresh reads catalogd /catalog/list, runs CREATE OR
REPLACE VIEW "name" AS SELECT * FROM read_parquet('s3://bucket/key')
per manifest, drops views for removed manifests, skips on unchanged
updated_at (the implicit etag). Drop pass runs BEFORE create pass so
a poison manifest can't block other manifest refreshes (post-scrum
C1 fix).
POST /sql with JSON body {"sql":"…"} returns
{"columns":[{"name":"id","type":"BIGINT"},…], "rows":[[…]],
"row_count":N}. []byte → string conversion so VARCHAR rows
JSON-encode as text. 30s default refresh ticker, configurable via
[queryd].refresh_every.
Cross-lineage scrum on shipped code:
- Opus 4.7 (opencode): 1 BLOCK + 4 WARN + 4 INFO
- Kimi K2-0905 (openrouter): 2 BLOCK + 2 WARN + 1 INFO
- Qwen3-coder (openrouter): 2 BLOCK + 1 WARN + 1 INFO
Fixed (4):
C1 (Opus + Kimi convergent): Refresh aborts on first per-view error
→ drop pass first, collect errors, errors.Join. Poison manifest
no longer blocks the rest of the catalog from re-syncing.
B-CTX (Opus BLOCK): bootstrap closure captured OpenDB's ctx →
cancelled-ctx silently fails every reconnect. context.Background()
inside closure; passed ctx only for initial Ping.
B-LEAK (Kimi BLOCK): firstLine(stmt) truncated CREATE SECRET to 80
chars but those 80 chars contained KEY_ID + SECRET prefix → log
aggregator captures credentials. Stable per-statement labels +
redactCreds() filter on wrapped DuckDB errors.
JSON-ERR (Opus WARN): swallowed json.Encode error → silent
truncated 200 on unsupported column types. slog.Warn the failure.
Dismissed (4 false positives):
Qwen BLOCK "bootstrap not transactional" — DuckDB DDL is auto-commit
Qwen BLOCK "MaxBytesReader after Decode" — false, applied before
Kimi BLOCK "concurrent Refresh + user SELECT deadlock" — not a
deadlock, just serialization, by design with 10s timeout retry
Kimi WARN "dropView leaves r.known inconsistent" — current code
returns before the delete; the entry persists for retry
Critical reviewer behavior: 1 convergent BLOCK between Opus + Kimi
on the per-view error blocking, plus two independent single-reviewer
BLOCKs (B-CTX, B-LEAK) that smoke could never have caught. The
B-LEAK fix uses defense-in-depth: never pass SQL into the error
path AND redact known cred values from DuckDB's own error message.
DuckDB cgo path: github.com/duckdb/duckdb-go/v2 v2.10502.0 (per
ADR-001 §1) on Go 1.25 + arrow-go. Smoke 6/6 PASS after every
fix round.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
+219
-16
@@ -1,22 +1,33 @@
|
||||
// queryd is the SQL execution layer — DuckDB via cgo, registers
|
||||
// catalog datasets as views, executes ad-hoc SQL. D5 wires the
|
||||
// actual /sql route + DuckDB session with S3 secret plumbed in
|
||||
// (per Opus B2 fix). D1 just stands up the binary.
|
||||
//
|
||||
// NOTE: cgo handle lifecycle. The shared server factory's
|
||||
// graceful-shutdown is enough for G0 (DuckDB connections are
|
||||
// goroutine-local), but G1+ when persistent prepared statements +
|
||||
// HNSW indexes attach, queryd will need its own OnShutdown drain
|
||||
// hook. Tracked in D7.5 (W3 disposition).
|
||||
// queryd is the SQL execution layer. DuckDB engine (via cgo), reads
|
||||
// Parquet directly from S3 via httpfs, registers catalog datasets as
|
||||
// views, exposes POST /sql. The interesting glue is in
|
||||
// internal/queryd/{db,registrar}.go; main.go wires the lifecycle.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.agentview.dev/profit/golangLAKEHOUSE/internal/shared"
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.agentview.dev/profit/golangLAKEHOUSE/internal/catalogclient"
|
||||
"git.agentview.dev/profit/golangLAKEHOUSE/internal/queryd"
|
||||
"git.agentview.dev/profit/golangLAKEHOUSE/internal/secrets"
|
||||
"git.agentview.dev/profit/golangLAKEHOUSE/internal/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
primaryBucket = "primary"
|
||||
maxSQLBodyBytes = 64 << 10 // 64 KiB cap on POST /sql body — SQL strings are not large
|
||||
defaultRefresh = 30 * time.Second
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -28,13 +39,205 @@ func main() {
|
||||
slog.Error("config", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg.Queryd.CatalogdURL == "" {
|
||||
slog.Error("config", "err", "queryd.catalogd_url is required")
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg.S3.Bucket == "" {
|
||||
slog.Error("config", "err", "s3.bucket is required")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := shared.Run("queryd", cfg.Queryd.Bind, func(_ chi.Router) {
|
||||
// D5 wires:
|
||||
// POST /sql (JSON body {"sql": "..."} → DuckDB exec → JSON rows)
|
||||
// Internal: TTL-cached views + etag invalidation against catalogd.
|
||||
}); err != nil {
|
||||
refreshEvery := defaultRefresh
|
||||
if cfg.Queryd.RefreshEvery != "" {
|
||||
d, err := time.ParseDuration(cfg.Queryd.RefreshEvery)
|
||||
if err != nil {
|
||||
slog.Error("config", "err", "queryd.refresh_every is not a valid duration: "+err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
refreshEvery = d
|
||||
}
|
||||
|
||||
// Long-running ctx tied to lifetime of the process for the
|
||||
// background refresh goroutine. Cancelled when shared.Run returns.
|
||||
procCtx, procCancel := context.WithCancel(context.Background())
|
||||
defer procCancel()
|
||||
|
||||
prov, err := secrets.NewFileProvider(cfg.Queryd.SecretsPath, secrets.S3Credentials{
|
||||
AccessKeyID: cfg.S3.AccessKeyID,
|
||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("secrets", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
db, err := queryd.OpenDB(procCtx, cfg.S3, prov, primaryBucket)
|
||||
if err != nil {
|
||||
slog.Error("duckdb open", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
catalog := catalogclient.New(cfg.Queryd.CatalogdURL)
|
||||
registrar := queryd.NewRegistrar(db, catalog, cfg.S3.Bucket)
|
||||
|
||||
// Initial refresh — log if non-empty so the operator sees what
|
||||
// got loaded. A failure here is non-fatal: catalogd may be coming
|
||||
// up later in the boot order, the TTL goroutine will retry.
|
||||
refreshCtx, refreshCancel := context.WithTimeout(procCtx, 10*time.Second)
|
||||
stats, err := registrar.Refresh(refreshCtx)
|
||||
refreshCancel()
|
||||
if err != nil {
|
||||
slog.Warn("initial refresh failed (will retry)", "err", err)
|
||||
} else {
|
||||
slog.Info("initial refresh", "created", stats.Created, "skipped", stats.Skipped)
|
||||
}
|
||||
|
||||
// Background ticker — drives Refresh on the configured interval.
|
||||
go runRefreshLoop(procCtx, registrar, refreshEvery)
|
||||
|
||||
h := &handlers{db: db}
|
||||
|
||||
if err := shared.Run("queryd", cfg.Queryd.Bind, h.register); err != nil {
|
||||
slog.Error("server", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// runRefreshLoop drives Registrar.Refresh on a ticker. Cancellable
|
||||
// via ctx. Logs every refresh; in a quiet run that's chatty but
|
||||
// useful for tracking when datasets land.
|
||||
func runRefreshLoop(ctx context.Context, r *queryd.Registrar, every time.Duration) {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
stats, err := r.Refresh(rctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
slog.Warn("refresh failed", "err", err)
|
||||
continue
|
||||
}
|
||||
if stats.Created+stats.Updated+stats.Dropped > 0 {
|
||||
slog.Info("refresh",
|
||||
"created", stats.Created,
|
||||
"updated", stats.Updated,
|
||||
"dropped", stats.Dropped,
|
||||
"skipped", stats.Skipped)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type handlers struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func (h *handlers) register(r chi.Router) {
|
||||
r.Post("/sql", h.handleSQL)
|
||||
}
|
||||
|
||||
// sqlRequest is the POST /sql body shape.
|
||||
type sqlRequest struct {
|
||||
SQL string `json:"sql"`
|
||||
}
|
||||
|
||||
// sqlResponse is the result envelope. Columns + rows is the compact
|
||||
// form; verbose row-as-object form is post-G0 if anyone needs it.
|
||||
type sqlResponse struct {
|
||||
Columns []sqlColumn `json:"columns"`
|
||||
Rows [][]any `json:"rows"`
|
||||
RowCount int `json:"row_count"`
|
||||
}
|
||||
|
||||
type sqlColumn struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
func (h *handlers) handleSQL(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxSQLBodyBytes)
|
||||
var req sqlRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) || strings.Contains(err.Error(), "http: request body too large") {
|
||||
http.Error(w, "sql body too large", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
http.Error(w, "decode body: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.SQL) == "" {
|
||||
http.Error(w, "sql is empty", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
rows, err := h.db.QueryContext(r.Context(), req.SQL)
|
||||
if err != nil {
|
||||
// DuckDB errors are user-facing for ad-hoc SQL — expose them
|
||||
// at 400 so callers can see what went wrong (table not found,
|
||||
// syntax error, etc.). Internal infra issues would surface as
|
||||
// 500 once we distinguish them later.
|
||||
http.Error(w, "query: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
http.Error(w, "columns: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
colTypes, err := rows.ColumnTypes()
|
||||
if err != nil {
|
||||
http.Error(w, "column types: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp := sqlResponse{
|
||||
Columns: make([]sqlColumn, len(cols)),
|
||||
Rows: [][]any{},
|
||||
}
|
||||
for i, name := range cols {
|
||||
resp.Columns[i] = sqlColumn{Name: name, Type: colTypes[i].DatabaseTypeName()}
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
dest := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range dest {
|
||||
ptrs[i] = &dest[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
http.Error(w, "scan: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// JSON can't encode []byte as text by default — DuckDB returns
|
||||
// VARCHAR as []byte through database/sql. Convert here.
|
||||
for i, v := range dest {
|
||||
if b, ok := v.([]byte); ok {
|
||||
dest[i] = string(b)
|
||||
}
|
||||
}
|
||||
resp.Rows = append(resp.Rows, dest)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
http.Error(w, "rows err: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
resp.RowCount = len(resp.Rows)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(resp); err != nil {
|
||||
// Headers already sent — can't change the status code, but log
|
||||
// so an unsupported DuckDB column type (Decimal, INTERVAL, etc.)
|
||||
// surfaces in the operator log. Per scrum JSON-ERR (Opus).
|
||||
slog.Warn("sql encode response", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user