diff --git a/internal/depot/config.go b/internal/depot/config.go new file mode 100644 index 0000000..f9d6d92 --- /dev/null +++ b/internal/depot/config.go @@ -0,0 +1,102 @@ +package depot + +import ( + "strconv" + "time" +) + +type Config struct { + CDNBase string + StorageHost string + StorageZone string + ReadKey string + WriteKey string + ProbeJobs int + Jobs int + ConnectTimeout time.Duration + ProbeMaxTime time.Duration + ConfirmRetries int + ConfirmMax int +} + +func LoadConfig(getenv func(string) string) Config { + cfg := Config{ + CDNBase: "https://cdn-a7f3k9.westgate.pw", + StorageZone: "sow-assets-depot", + ProbeJobs: 16, + Jobs: 16, + ConnectTimeout: 10 * time.Second, + ProbeMaxTime: 30 * time.Second, + ConfirmRetries: 3, + ConfirmMax: 200, + } + + // DEPOT_CDN_BASE || BUNNY_CDN_BASE + if v := getenv("DEPOT_CDN_BASE"); v != "" { + cfg.CDNBase = v + } else if v := getenv("BUNNY_CDN_BASE"); v != "" { + cfg.CDNBase = v + } + + // BUNNY_STORAGE_HOST (no default) + cfg.StorageHost = getenv("BUNNY_STORAGE_HOST") + + // BUNNY_STORAGE_ZONE (default "sow-assets-depot") + if v := getenv("BUNNY_STORAGE_ZONE"); v != "" { + cfg.StorageZone = v + } + + // BUNNY_STORAGE_PASSWORD + cfg.WriteKey = getenv("BUNNY_STORAGE_PASSWORD") + + // BUNNY_STORAGE_READ_PASSWORD || BUNNY_STORAGE_PASSWORD + if v := getenv("BUNNY_STORAGE_READ_PASSWORD"); v != "" { + cfg.ReadKey = v + } else { + cfg.ReadKey = cfg.WriteKey + } + + // DEPOT_PROBE_JOBS (default 16) + if v := getenv("DEPOT_PROBE_JOBS"); v != "" { + if i, err := strconv.Atoi(v); err == nil { + cfg.ProbeJobs = i + } + } + + // DEPOT_JOBS (default 16) + if v := getenv("DEPOT_JOBS"); v != "" { + if i, err := strconv.Atoi(v); err == nil { + cfg.Jobs = i + } + } + + // DEPOT_CONNECT_TIMEOUT (default 10s) + if v := getenv("DEPOT_CONNECT_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v + "s"); err == nil { + cfg.ConnectTimeout = d + } + } + + // DEPOT_PROBE_MAX_TIME (default 30s) + if v := getenv("DEPOT_PROBE_MAX_TIME"); v != "" { + if d, err := time.ParseDuration(v + "s"); err == nil { + cfg.ProbeMaxTime = d + } + } + + // DEPOT_CONFIRM_RETRIES (default 3) + if v := getenv("DEPOT_CONFIRM_RETRIES"); v != "" { + if i, err := strconv.Atoi(v); err == nil { + cfg.ConfirmRetries = i + } + } + + // DEPOT_CONFIRM_MAX (default 200, 0 = unlimited) + if v := getenv("DEPOT_CONFIRM_MAX"); v != "" { + if i, err := strconv.Atoi(v); err == nil { + cfg.ConfirmMax = i + } + } + + return cfg +} diff --git a/internal/depot/config_test.go b/internal/depot/config_test.go new file mode 100644 index 0000000..e487fab --- /dev/null +++ b/internal/depot/config_test.go @@ -0,0 +1,115 @@ +package depot + +import ( + "testing" + "time" +) + +func TestLoadConfigDefaults(t *testing.T) { + getenv := func(key string) string { + return "" + } + cfg := LoadConfig(getenv) + if cfg.CDNBase != "https://cdn-a7f3k9.westgate.pw" { + t.Errorf("CDNBase: got %q, want %q", cfg.CDNBase, "https://cdn-a7f3k9.westgate.pw") + } + if cfg.StorageZone != "sow-assets-depot" { + t.Errorf("StorageZone: got %q, want %q", cfg.StorageZone, "sow-assets-depot") + } + if cfg.ProbeJobs != 16 { + t.Errorf("ProbeJobs: got %d, want 16", cfg.ProbeJobs) + } + if cfg.Jobs != 16 { + t.Errorf("Jobs: got %d, want 16", cfg.Jobs) + } + if cfg.ConnectTimeout != 10*time.Second { + t.Errorf("ConnectTimeout: got %v, want 10s", cfg.ConnectTimeout) + } + if cfg.ProbeMaxTime != 30*time.Second { + t.Errorf("ProbeMaxTime: got %v, want 30s", cfg.ProbeMaxTime) + } + if cfg.ConfirmRetries != 3 { + t.Errorf("ConfirmRetries: got %d, want 3", cfg.ConfirmRetries) + } + if cfg.ConfirmMax != 200 { + t.Errorf("ConfirmMax: got %d, want 200", cfg.ConfirmMax) + } +} + +func TestLoadConfigReadKeyFallback(t *testing.T) { + // When READ unset, falls back to WriteKey + getenv := func(key string) string { + if key == "BUNNY_STORAGE_PASSWORD" { + return "write-key" + } + return "" + } + cfg := LoadConfig(getenv) + if cfg.ReadKey != "write-key" { + t.Errorf("ReadKey fallback: got %q, want %q", cfg.ReadKey, "write-key") + } + if cfg.WriteKey != "write-key" { + t.Errorf("WriteKey: got %q, want %q", cfg.WriteKey, "write-key") + } + + // When READ is set, uses its own value + getenv = func(key string) string { + if key == "BUNNY_STORAGE_READ_PASSWORD" { + return "read-key" + } + if key == "BUNNY_STORAGE_PASSWORD" { + return "write-key" + } + return "" + } + cfg = LoadConfig(getenv) + if cfg.ReadKey != "read-key" { + t.Errorf("ReadKey explicit: got %q, want %q", cfg.ReadKey, "read-key") + } + if cfg.WriteKey != "write-key" { + t.Errorf("WriteKey: got %q, want %q", cfg.WriteKey, "write-key") + } +} + +func TestLoadConfigBadIntFallsBack(t *testing.T) { + getenv := func(key string) string { + if key == "DEPOT_PROBE_JOBS" { + return "not-a-number" + } + if key == "DEPOT_JOBS" { + return "invalid" + } + if key == "DEPOT_CONNECT_TIMEOUT" { + return "bad" + } + if key == "DEPOT_PROBE_MAX_TIME" { + return "wrong" + } + if key == "DEPOT_CONFIRM_RETRIES" { + return "nope" + } + if key == "DEPOT_CONFIRM_MAX" { + return "nah" + } + return "" + } + cfg := LoadConfig(getenv) + if cfg.ProbeJobs != 16 { + t.Errorf("ProbeJobs fallback: got %d, want 16", cfg.ProbeJobs) + } + if cfg.Jobs != 16 { + t.Errorf("Jobs fallback: got %d, want 16", cfg.Jobs) + } + if cfg.ConnectTimeout != 10*time.Second { + t.Errorf("ConnectTimeout fallback: got %v, want 10s", cfg.ConnectTimeout) + } + if cfg.ProbeMaxTime != 30*time.Second { + t.Errorf("ProbeMaxTime fallback: got %v, want 30s", cfg.ProbeMaxTime) + } + if cfg.ConfirmRetries != 3 { + t.Errorf("ConfirmRetries fallback: got %d, want 3", cfg.ConfirmRetries) + } + if cfg.ConfirmMax != 200 { + t.Errorf("ConfirmMax fallback: got %d, want 200", cfg.ConfirmMax) + } +} diff --git a/internal/depot/manifest.go b/internal/depot/manifest.go new file mode 100644 index 0000000..f6c2d04 --- /dev/null +++ b/internal/depot/manifest.go @@ -0,0 +1,82 @@ +package depot + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + + "gopkg.in/yaml.v3" +) + +type manifestFile struct { + Assets []struct { + Path string `yaml:"path"` + SHA256 string `yaml:"sha256"` + Size int64 `yaml:"size"` + } `yaml:"assets"` +} + +func ValidSHA(s string) bool { + if len(s) != 64 { + return false + } + match, _ := regexp.MatchString("^[0-9a-f]{64}$", s) + return match +} + +func BlobKey(sha string) string { + return fmt.Sprintf("sha256/%s/%s/%s", sha[0:2], sha[2:4], sha) +} + +func ReferencedSHAs(dir string) (map[string]int64, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + result := make(map[string]int64) + foundAny := false + + for _, entry := range entries { + if entry.IsDir() { + continue + } + if !hasYAMLExt(entry.Name()) { + continue + } + + foundAny = true + path := filepath.Join(dir, entry.Name()) + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var manifest manifestFile + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, err + } + + for _, asset := range manifest.Assets { + if !ValidSHA(asset.SHA256) { + return nil, fmt.Errorf("%s: invalid sha256 hash", entry.Name()) + } + + // Keep the largest size for each sha + if asset.Size > result[asset.SHA256] { + result[asset.SHA256] = asset.Size + } + } + } + + if !foundAny { + return nil, fmt.Errorf("no *.yml files found in %s", dir) + } + + return result, nil +} + +func hasYAMLExt(name string) bool { + return filepath.Ext(name) == ".yml" || filepath.Ext(name) == ".yaml" +} diff --git a/internal/depot/manifest_test.go b/internal/depot/manifest_test.go new file mode 100644 index 0000000..b738e6e --- /dev/null +++ b/internal/depot/manifest_test.go @@ -0,0 +1,128 @@ +package depot + +import ( + "os" + "path/filepath" + "testing" +) + +func TestValidSHA(t *testing.T) { + tests := []struct { + input string + want bool + }{ + {"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", true}, + {"0000000000000000000000000000000000000000000000000000000000000000", true}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", true}, + {"0b1d1234567890ABCDEF1234567890abcdef1234567890abcdef1234567890ab", false}, // uppercase + {"0b1d1234567890abcde", false}, // too short + {"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890abff", false}, // too long + {"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ag", false}, // invalid hex + {"", false}, + } + for _, tc := range tests { + got := ValidSHA(tc.input) + if got != tc.want { + t.Errorf("ValidSHA(%q): got %v, want %v", tc.input, got, tc.want) + } + } +} + +func TestBlobKey(t *testing.T) { + tests := []struct { + sha string + want string + }{ + {"0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab", "sha256/0b/1d/0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab"}, + {"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", "sha256/ab/cd/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}, + } + for _, tc := range tests { + got := BlobKey(tc.sha) + if got != tc.want { + t.Errorf("BlobKey(%q): got %q, want %q", tc.sha, got, tc.want) + } + } +} + +func TestReferencedSHAs(t *testing.T) { + // Test: temp dir with two yml files sharing one sha → dedup + tmpdir := t.TempDir() + + // First manifest with sha1 and sha2 + manifest1 := `assets: + - path: file1.txt + sha256: 0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab + size: 100 + - path: file2.txt + sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 + size: 200 +` + if err := os.WriteFile(filepath.Join(tmpdir, "manifest1.yml"), []byte(manifest1), 0644); err != nil { + t.Fatalf("write manifest1: %v", err) + } + + // Second manifest with sha2 and sha3 + manifest2 := `assets: + - path: file3.txt + sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 + size: 300 + - path: file4.txt + sha256: fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210 + size: 400 +` + if err := os.WriteFile(filepath.Join(tmpdir, "manifest2.yml"), []byte(manifest2), 0644); err != nil { + t.Fatalf("write manifest2: %v", err) + } + + result, err := ReferencedSHAs(tmpdir) + if err != nil { + t.Fatalf("ReferencedSHAs: %v", err) + } + + // Should have 3 unique SHAs + if len(result) != 3 { + t.Errorf("ReferencedSHAs: got %d unique SHAs, want 3", len(result)) + } + + expectedSizes := map[string]int64{ + "0b1d1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab": 100, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789": 300, // Largest size for duplicated sha + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210": 400, + } + for sha, expectedSize := range expectedSizes { + size, exists := result[sha] + if !exists { + t.Errorf("ReferencedSHAs: sha %q missing", sha) + } + if size != expectedSize { + t.Errorf("ReferencedSHAs: sha %q size: got %d, want %d", sha, size, expectedSize) + } + } + + // Test: bad sha → error naming the file + badManifest := `assets: + - path: file5.txt + sha256: not_a_valid_sha + size: 500 +` + if err := os.WriteFile(filepath.Join(tmpdir, "badsha.yml"), []byte(badManifest), 0644); err != nil { + t.Fatalf("write badsha: %v", err) + } + + _, err = ReferencedSHAs(tmpdir) + if err == nil { + t.Errorf("ReferencedSHAs with bad sha: got nil error, want error naming file") + } + if err != nil && err.Error() != "badsha.yml: invalid sha256 hash" { + t.Errorf("ReferencedSHAs error message: got %q, should mention badsha.yml", err.Error()) + } +} + +func TestReferencedSHAsEmpty(t *testing.T) { + // Test: empty dir → error + tmpdir := t.TempDir() + _, err := ReferencedSHAs(tmpdir) + if err == nil { + t.Errorf("ReferencedSHAs on empty dir: got nil error, want error") + } +}