feat(depot): config resolution and manifest sha parsing
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,102 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
CDNBase string
|
||||
StorageHost string
|
||||
StorageZone string
|
||||
ReadKey string
|
||||
WriteKey string
|
||||
ProbeJobs int
|
||||
Jobs int
|
||||
ConnectTimeout time.Duration
|
||||
ProbeMaxTime time.Duration
|
||||
ConfirmRetries int
|
||||
ConfirmMax int
|
||||
}
|
||||
|
||||
func LoadConfig(getenv func(string) string) Config {
|
||||
cfg := Config{
|
||||
CDNBase: "https://cdn-a7f3k9.westgate.pw",
|
||||
StorageZone: "sow-assets-depot",
|
||||
ProbeJobs: 16,
|
||||
Jobs: 16,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
ProbeMaxTime: 30 * time.Second,
|
||||
ConfirmRetries: 3,
|
||||
ConfirmMax: 200,
|
||||
}
|
||||
|
||||
// DEPOT_CDN_BASE || BUNNY_CDN_BASE
|
||||
if v := getenv("DEPOT_CDN_BASE"); v != "" {
|
||||
cfg.CDNBase = v
|
||||
} else if v := getenv("BUNNY_CDN_BASE"); v != "" {
|
||||
cfg.CDNBase = v
|
||||
}
|
||||
|
||||
// BUNNY_STORAGE_HOST (no default)
|
||||
cfg.StorageHost = getenv("BUNNY_STORAGE_HOST")
|
||||
|
||||
// BUNNY_STORAGE_ZONE (default "sow-assets-depot")
|
||||
if v := getenv("BUNNY_STORAGE_ZONE"); v != "" {
|
||||
cfg.StorageZone = v
|
||||
}
|
||||
|
||||
// BUNNY_STORAGE_PASSWORD
|
||||
cfg.WriteKey = getenv("BUNNY_STORAGE_PASSWORD")
|
||||
|
||||
// BUNNY_STORAGE_READ_PASSWORD || BUNNY_STORAGE_PASSWORD
|
||||
if v := getenv("BUNNY_STORAGE_READ_PASSWORD"); v != "" {
|
||||
cfg.ReadKey = v
|
||||
} else {
|
||||
cfg.ReadKey = cfg.WriteKey
|
||||
}
|
||||
|
||||
// DEPOT_PROBE_JOBS (default 16)
|
||||
if v := getenv("DEPOT_PROBE_JOBS"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.ProbeJobs = i
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_JOBS (default 16)
|
||||
if v := getenv("DEPOT_JOBS"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.Jobs = i
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_CONNECT_TIMEOUT (default 10s)
|
||||
if v := getenv("DEPOT_CONNECT_TIMEOUT"); v != "" {
|
||||
if d, err := time.ParseDuration(v + "s"); err == nil {
|
||||
cfg.ConnectTimeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_PROBE_MAX_TIME (default 30s)
|
||||
if v := getenv("DEPOT_PROBE_MAX_TIME"); v != "" {
|
||||
if d, err := time.ParseDuration(v + "s"); err == nil {
|
||||
cfg.ProbeMaxTime = d
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_CONFIRM_RETRIES (default 3)
|
||||
if v := getenv("DEPOT_CONFIRM_RETRIES"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.ConfirmRetries = i
|
||||
}
|
||||
}
|
||||
|
||||
// DEPOT_CONFIRM_MAX (default 200, 0 = unlimited)
|
||||
if v := getenv("DEPOT_CONFIRM_MAX"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
cfg.ConfirmMax = i
|
||||
}
|
||||
}
|
||||
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadConfigDefaults(t *testing.T) {
|
||||
getenv := func(key string) string {
|
||||
return ""
|
||||
}
|
||||
cfg := LoadConfig(getenv)
|
||||
if cfg.CDNBase != "https://cdn-a7f3k9.westgate.pw" {
|
||||
t.Errorf("CDNBase: got %q, want %q", cfg.CDNBase, "https://cdn-a7f3k9.westgate.pw")
|
||||
}
|
||||
if cfg.StorageZone != "sow-assets-depot" {
|
||||
t.Errorf("StorageZone: got %q, want %q", cfg.StorageZone, "sow-assets-depot")
|
||||
}
|
||||
if cfg.ProbeJobs != 16 {
|
||||
t.Errorf("ProbeJobs: got %d, want 16", cfg.ProbeJobs)
|
||||
}
|
||||
if cfg.Jobs != 16 {
|
||||
t.Errorf("Jobs: got %d, want 16", cfg.Jobs)
|
||||
}
|
||||
if cfg.ConnectTimeout != 10*time.Second {
|
||||
t.Errorf("ConnectTimeout: got %v, want 10s", cfg.ConnectTimeout)
|
||||
}
|
||||
if cfg.ProbeMaxTime != 30*time.Second {
|
||||
t.Errorf("ProbeMaxTime: got %v, want 30s", cfg.ProbeMaxTime)
|
||||
}
|
||||
if cfg.ConfirmRetries != 3 {
|
||||
t.Errorf("ConfirmRetries: got %d, want 3", cfg.ConfirmRetries)
|
||||
}
|
||||
if cfg.ConfirmMax != 200 {
|
||||
t.Errorf("ConfirmMax: got %d, want 200", cfg.ConfirmMax)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadKeyFallback(t *testing.T) {
|
||||
// When READ unset, falls back to WriteKey
|
||||
getenv := func(key string) string {
|
||||
if key == "BUNNY_STORAGE_PASSWORD" {
|
||||
return "write-key"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
cfg := LoadConfig(getenv)
|
||||
if cfg.ReadKey != "write-key" {
|
||||
t.Errorf("ReadKey fallback: got %q, want %q", cfg.ReadKey, "write-key")
|
||||
}
|
||||
if cfg.WriteKey != "write-key" {
|
||||
t.Errorf("WriteKey: got %q, want %q", cfg.WriteKey, "write-key")
|
||||
}
|
||||
|
||||
// When READ is set, uses its own value
|
||||
getenv = func(key string) string {
|
||||
if key == "BUNNY_STORAGE_READ_PASSWORD" {
|
||||
return "read-key"
|
||||
}
|
||||
if key == "BUNNY_STORAGE_PASSWORD" {
|
||||
return "write-key"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
cfg = LoadConfig(getenv)
|
||||
if cfg.ReadKey != "read-key" {
|
||||
t.Errorf("ReadKey explicit: got %q, want %q", cfg.ReadKey, "read-key")
|
||||
}
|
||||
if cfg.WriteKey != "write-key" {
|
||||
t.Errorf("WriteKey: got %q, want %q", cfg.WriteKey, "write-key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigBadIntFallsBack(t *testing.T) {
|
||||
getenv := func(key string) string {
|
||||
if key == "DEPOT_PROBE_JOBS" {
|
||||
return "not-a-number"
|
||||
}
|
||||
if key == "DEPOT_JOBS" {
|
||||
return "invalid"
|
||||
}
|
||||
if key == "DEPOT_CONNECT_TIMEOUT" {
|
||||
return "bad"
|
||||
}
|
||||
if key == "DEPOT_PROBE_MAX_TIME" {
|
||||
return "wrong"
|
||||
}
|
||||
if key == "DEPOT_CONFIRM_RETRIES" {
|
||||
return "nope"
|
||||
}
|
||||
if key == "DEPOT_CONFIRM_MAX" {
|
||||
return "nah"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
cfg := LoadConfig(getenv)
|
||||
if cfg.ProbeJobs != 16 {
|
||||
t.Errorf("ProbeJobs fallback: got %d, want 16", cfg.ProbeJobs)
|
||||
}
|
||||
if cfg.Jobs != 16 {
|
||||
t.Errorf("Jobs fallback: got %d, want 16", cfg.Jobs)
|
||||
}
|
||||
if cfg.ConnectTimeout != 10*time.Second {
|
||||
t.Errorf("ConnectTimeout fallback: got %v, want 10s", cfg.ConnectTimeout)
|
||||
}
|
||||
if cfg.ProbeMaxTime != 30*time.Second {
|
||||
t.Errorf("ProbeMaxTime fallback: got %v, want 30s", cfg.ProbeMaxTime)
|
||||
}
|
||||
if cfg.ConfirmRetries != 3 {
|
||||
t.Errorf("ConfirmRetries fallback: got %d, want 3", cfg.ConfirmRetries)
|
||||
}
|
||||
if cfg.ConfirmMax != 200 {
|
||||
t.Errorf("ConfirmMax fallback: got %d, want 200", cfg.ConfirmMax)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type manifestFile struct {
|
||||
Assets []struct {
|
||||
Path string `yaml:"path"`
|
||||
SHA256 string `yaml:"sha256"`
|
||||
Size int64 `yaml:"size"`
|
||||
} `yaml:"assets"`
|
||||
}
|
||||
|
||||
func ValidSHA(s string) bool {
|
||||
if len(s) != 64 {
|
||||
return false
|
||||
}
|
||||
match, _ := regexp.MatchString("^[0-9a-f]{64}$", s)
|
||||
return match
|
||||
}
|
||||
|
||||
func BlobKey(sha string) string {
|
||||
return fmt.Sprintf("sha256/%s/%s/%s", sha[0:2], sha[2:4], sha)
|
||||
}
|
||||
|
||||
func ReferencedSHAs(dir string) (map[string]int64, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make(map[string]int64)
|
||||
foundAny := false
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if !hasYAMLExt(entry.Name()) {
|
||||
continue
|
||||
}
|
||||
|
||||
foundAny = true
|
||||
path := filepath.Join(dir, entry.Name())
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var manifest manifestFile
|
||||
if err := yaml.Unmarshal(data, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, asset := range manifest.Assets {
|
||||
if !ValidSHA(asset.SHA256) {
|
||||
return nil, fmt.Errorf("%s: invalid sha256 hash", entry.Name())
|
||||
}
|
||||
|
||||
// Keep the largest size for each sha
|
||||
if asset.Size > result[asset.SHA256] {
|
||||
result[asset.SHA256] = asset.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !foundAny {
|
||||
return nil, fmt.Errorf("no *.yml files found in %s", dir)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func hasYAMLExt(name string) bool {
|
||||
return filepath.Ext(name) == ".yml" || filepath.Ext(name) == ".yaml"
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidSHA(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", true},
|
||||
{"0000000000000000000000000000000000000000000000000000000000000000", true},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", true},
|
||||
{"0b1d1234567890ABCDEF1234567890abcdef1234567890abcdef1234567890ab", false}, // uppercase
|
||||
{"0b1d1234567890abcde", false}, // too short
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890abff", false}, // too long
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ag", false}, // invalid hex
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := ValidSHA(tc.input)
|
||||
if got != tc.want {
|
||||
t.Errorf("ValidSHA(%q): got %v, want %v", tc.input, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
sha string
|
||||
want string
|
||||
}{
|
||||
{"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", "sha256/0b/1d/0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab"},
|
||||
{"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", "sha256/ab/cd/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := BlobKey(tc.sha)
|
||||
if got != tc.want {
|
||||
t.Errorf("BlobKey(%q): got %q, want %q", tc.sha, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencedSHAs(t *testing.T) {
|
||||
// Test: temp dir with two yml files sharing one sha → dedup
|
||||
tmpdir := t.TempDir()
|
||||
|
||||
// First manifest with sha1 and sha2
|
||||
manifest1 := `assets:
|
||||
- path: file1.txt
|
||||
sha256: 0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab
|
||||
size: 100
|
||||
- path: file2.txt
|
||||
sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789
|
||||
size: 200
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(tmpdir, "manifest1.yml"), []byte(manifest1), 0644); err != nil {
|
||||
t.Fatalf("write manifest1: %v", err)
|
||||
}
|
||||
|
||||
// Second manifest with sha2 and sha3
|
||||
manifest2 := `assets:
|
||||
- path: file3.txt
|
||||
sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789
|
||||
size: 300
|
||||
- path: file4.txt
|
||||
sha256: fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210
|
||||
size: 400
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(tmpdir, "manifest2.yml"), []byte(manifest2), 0644); err != nil {
|
||||
t.Fatalf("write manifest2: %v", err)
|
||||
}
|
||||
|
||||
result, err := ReferencedSHAs(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReferencedSHAs: %v", err)
|
||||
}
|
||||
|
||||
// Should have 3 unique SHAs
|
||||
if len(result) != 3 {
|
||||
t.Errorf("ReferencedSHAs: got %d unique SHAs, want 3", len(result))
|
||||
}
|
||||
|
||||
expectedSizes := map[string]int64{
|
||||
"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab": 100,
|
||||
"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789": 300, // Largest size for duplicated sha
|
||||
"fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210": 400,
|
||||
}
|
||||
for sha, expectedSize := range expectedSizes {
|
||||
size, exists := result[sha]
|
||||
if !exists {
|
||||
t.Errorf("ReferencedSHAs: sha %q missing", sha)
|
||||
}
|
||||
if size != expectedSize {
|
||||
t.Errorf("ReferencedSHAs: sha %q size: got %d, want %d", sha, size, expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Test: bad sha → error naming the file
|
||||
badManifest := `assets:
|
||||
- path: file5.txt
|
||||
sha256: not_a_valid_sha
|
||||
size: 500
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(tmpdir, "badsha.yml"), []byte(badManifest), 0644); err != nil {
|
||||
t.Fatalf("write badsha: %v", err)
|
||||
}
|
||||
|
||||
_, err = ReferencedSHAs(tmpdir)
|
||||
if err == nil {
|
||||
t.Errorf("ReferencedSHAs with bad sha: got nil error, want error naming file")
|
||||
}
|
||||
if err != nil && err.Error() != "badsha.yml: invalid sha256 hash" {
|
||||
t.Errorf("ReferencedSHAs error message: got %q, should mention badsha.yml", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencedSHAsEmpty(t *testing.T) {
|
||||
// Test: empty dir → error
|
||||
tmpdir := t.TempDir()
|
||||
_, err := ReferencedSHAs(tmpdir)
|
||||
if err == nil {
|
||||
t.Errorf("ReferencedSHAs on empty dir: got nil error, want error")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user