G0 D2: storaged S3 GET/PUT/LIST/DELETE · 3-lineage scrum · 4 fixes applied
Phase G0 Day 2 ships storaged: aws-sdk-go-v2 wrapper + chi routes
binding 127.0.0.1:3211 with 256 MiB MaxBytesReader, Content-Length
up-front 413, and a 4-slot non-blocking semaphore returning 503 +
Retry-After:5 when full. Acceptance smoke (6/6 probes) PASSES against
the dedicated MinIO bucket lakehouse-go-primary, isolated from the
Rust system's lakehouse bucket during coexistence.
Cross-lineage scrum on the shipped code:
- Opus 4.7 (opencode): 1 BLOCK + 3 WARN + 3 INFO
- Qwen3-coder (openrouter): 2 BLOCK + 1 WARN + 1 INFO (3 false positives)
- Kimi K2-0905 (openrouter, after route-shopping past opencode's 4k
cap and the direct adapter's empty-content reasoning bug):
1 BLOCK + 2 WARN + 1 INFO
Fixed:
C1 buildRegistry ctx cancel footgun → context.Background()
(Opus + Kimi convergent; future credential refresh chains)
C2 MaxBytesReader unwrap through manager.Uploader multipart
goroutines → Content-Length up-front 413 + string-suffix fallback
(Opus + Kimi convergent; latent 500-instead-of-413 in 5-256 MiB range)
C3 Bucket.List unbounded accumulation → MaxListResults=10_000 cap
(Opus + Kimi convergent; OOM guard)
S1 PUT response Content-Type: application/json (Opus single-reviewer)
Strict validateKey policy (J approved): rejects empty, >1024B, NUL,
leading "/", ".." path components, CR/LF/tab control characters.
DELETE exposed at HTTP layer (J approved option A) for symmetry +
smoke ergonomics.
Build clean, vet clean, all unit tests pass, smoke 6/6 PASS after
every fix round. go.mod 1.23 → 1.24 (required by aws-sdk-go-v2).
Process finding worth recording: opencode caps non-streaming Kimi at
max_tokens=4096; the direct kimi.com adapter consumed 8192 tokens of
reasoning but surfaced empty content; openrouter/moonshotai/kimi-k2-0905
delivered structured output in ~33s. Future Kimi scrums should default
to that route.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -0,0 +1,108 @@
|
||||
// Package secrets resolves credentials for storaged + future bucket
|
||||
// federation (G2). The G0 surface is one method — S3Credentials —
|
||||
// looked up by logical bucket name. Multi-bucket lands in G2; until
|
||||
// then every lookup returns the same credentials, but callers already
|
||||
// pass the name so the API doesn't need to change later.
|
||||
//
|
||||
// FileProvider reads /etc/lakehouse/secrets.toml. If that file is
|
||||
// absent OR doesn't contain credentials for the requested bucket, the
|
||||
// provider falls back to the values supplied via the inline config
|
||||
// (lakehouse.toml [s3] block). G0 is dev-only so the inline fallback
|
||||
// is convenient; G1 will tighten this to "secrets file required".
|
||||
package secrets
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/pelletier/go-toml/v2"
|
||||
)
|
||||
|
||||
// S3Credentials is what storaged hands to aws-sdk-go-v2 to sign
|
||||
// requests. Region/endpoint/bucket are config (non-secret) and live
|
||||
// on shared.S3Config — those don't pass through this provider.
|
||||
type S3Credentials struct {
|
||||
AccessKeyID string `toml:"access_key_id"`
|
||||
SecretAccessKey string `toml:"secret_access_key"`
|
||||
}
|
||||
|
||||
// Provider is the interface storaged depends on. Keeping this small
|
||||
// is deliberate — every method here is a future migration point when
|
||||
// secrets move to Vault / SOPS / SSM.
|
||||
type Provider interface {
|
||||
S3Credentials(bucket string) (S3Credentials, error)
|
||||
}
|
||||
|
||||
// FileProvider is the G0 implementation. It loads the file once on
|
||||
// construction; reload-on-SIGHUP is a G1 concern.
|
||||
type FileProvider struct {
|
||||
path string
|
||||
parsed secretsFile
|
||||
fallback S3Credentials
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// secretsFile mirrors the on-disk TOML shape:
|
||||
//
|
||||
// [s3.primary]
|
||||
// access_key_id = "..."
|
||||
// secret_access_key = "..."
|
||||
//
|
||||
// [s3.archive] # G2 multi-bucket
|
||||
// access_key_id = "..."
|
||||
type secretsFile struct {
|
||||
S3 map[string]S3Credentials `toml:"s3"`
|
||||
}
|
||||
|
||||
// NewFileProvider loads `path`. If the file is missing the provider
|
||||
// is still usable with the inline fallback — that's a deliberate
|
||||
// G0 affordance. Any other read/parse error is fatal.
|
||||
func NewFileProvider(path string, fallback S3Credentials) (*FileProvider, error) {
|
||||
p := &FileProvider{path: path, fallback: fallback}
|
||||
b, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return p, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read secrets %q: %w", path, err)
|
||||
}
|
||||
if err := toml.Unmarshal(b, &p.parsed); err != nil {
|
||||
return nil, fmt.Errorf("parse secrets %q: %w", path, err)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// S3Credentials resolves bucket → credentials. Lookup order:
|
||||
// 1. secrets file [s3.<bucket>] section
|
||||
// 2. inline fallback (lakehouse.toml [s3])
|
||||
//
|
||||
// If neither produces a non-empty AccessKeyID, returns an error so a
|
||||
// misconfigured deployment fails loud instead of trying anonymous S3.
|
||||
func (p *FileProvider) S3Credentials(bucket string) (S3Credentials, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
|
||||
if creds, ok := p.parsed.S3[bucket]; ok && creds.AccessKeyID != "" {
|
||||
return creds, nil
|
||||
}
|
||||
if p.fallback.AccessKeyID != "" {
|
||||
return p.fallback, nil
|
||||
}
|
||||
return S3Credentials{}, fmt.Errorf("no credentials for bucket %q (file=%q)", bucket, p.path)
|
||||
}
|
||||
|
||||
// StaticProvider is a test/dev helper that returns the same creds for
|
||||
// every bucket. Use NewFileProvider in production code paths.
|
||||
type StaticProvider struct {
|
||||
Creds S3Credentials
|
||||
}
|
||||
|
||||
func (p StaticProvider) S3Credentials(_ string) (S3Credentials, error) {
|
||||
if p.Creds.AccessKeyID == "" {
|
||||
return S3Credentials{}, errors.New("StaticProvider: no AccessKeyID")
|
||||
}
|
||||
return p.Creds, nil
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package secrets
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileProvider_ParsesSection(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "secrets.toml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
[s3.primary]
|
||||
access_key_id = "AK"
|
||||
secret_access_key = "SK"
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := NewFileProvider(path, S3Credentials{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileProvider: %v", err)
|
||||
}
|
||||
got, err := p.S3Credentials("primary")
|
||||
if err != nil {
|
||||
t.Fatalf("S3Credentials: %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "AK" || got.SecretAccessKey != "SK" {
|
||||
t.Errorf("got %+v, want {AK, SK}", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProvider_FallbackWhenSectionMissing(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "secrets.toml")
|
||||
// File exists, but doesn't have an [s3.primary] block.
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
[s3.archive]
|
||||
access_key_id = "OTHER"
|
||||
secret_access_key = "OTHER_SK"
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := NewFileProvider(path, S3Credentials{
|
||||
AccessKeyID: "FALLBACK",
|
||||
SecretAccessKey: "FALLBACK_SK",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := p.S3Credentials("primary")
|
||||
if err != nil {
|
||||
t.Fatalf("S3Credentials: %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "FALLBACK" {
|
||||
t.Errorf("expected fallback, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProvider_MissingFileIsOK(t *testing.T) {
|
||||
p, err := NewFileProvider("/no/such/path", S3Credentials{
|
||||
AccessKeyID: "FALLBACK",
|
||||
SecretAccessKey: "FALLBACK_SK",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileProvider should not error on missing file: %v", err)
|
||||
}
|
||||
got, err := p.S3Credentials("primary")
|
||||
if err != nil {
|
||||
t.Fatalf("S3Credentials: %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "FALLBACK" {
|
||||
t.Errorf("expected fallback, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProvider_NoCredsAtAll(t *testing.T) {
|
||||
p, err := NewFileProvider("/no/such/path", S3Credentials{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := p.S3Credentials("primary"); err == nil {
|
||||
t.Fatal("expected error when no creds in file or fallback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileProvider_ParseError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.toml")
|
||||
if err := os.WriteFile(path, []byte("not valid toml ===\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := NewFileProvider(path, S3Credentials{}); err == nil {
|
||||
t.Fatal("expected parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaticProvider(t *testing.T) {
|
||||
p := StaticProvider{Creds: S3Credentials{AccessKeyID: "X", SecretAccessKey: "Y"}}
|
||||
got, err := p.S3Credentials("any-bucket")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.AccessKeyID != "X" {
|
||||
t.Errorf("got %+v", got)
|
||||
}
|
||||
|
||||
empty := StaticProvider{}
|
||||
if _, err := empty.S3Credentials("any"); err == nil {
|
||||
t.Error("expected error from empty StaticProvider")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaticProvider_ErrorIsExported(t *testing.T) {
|
||||
// Sanity: the empty-creds path should be a real error type, not nil.
|
||||
_, err := StaticProvider{}.S3Credentials("x")
|
||||
if err == nil || !errors.Is(err, err) {
|
||||
t.Fatal("expected non-nil error")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user