diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go index 37dc846..fd4494d 100644 --- a/internal/topdata/autogen.go +++ b/internal/topdata/autogen.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "os" "path/filepath" "slices" @@ -13,10 +14,13 @@ import ( "time" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gopkg.in/yaml.v3" ) const autogenManifestCacheMaxAge = time.Hour +const defaultBunnyCDNBase = "https://depot.westgate.pw" + // errAutogenManifestUnavailable marks a released autogen manifest that could not // be located or fetched (network failure, missing release/asset, empty payload, // or an undeterminable sow-assets repo). It is deliberately distinct from an @@ -225,6 +229,9 @@ func autogenConsumerTargetsDataset(dataset nativeCollectedDataset, consumer proj } func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) { + if strings.TrimSpace(consumer.Source.Kind) == "cdn_channel" { + return resolveCDNChannelManifest(p, consumer, consumer.Source, progress) + } if manifestFile := strings.TrimSpace(consumer.ManifestFile); manifestFile != "" { return readAutogenConsumerManifestFile(p, consumer, manifestFile, progress) } @@ -276,6 +283,183 @@ func readAutogenConsumerManifestFile(p *project.Project, consumer project.Autoge return &manifest, nil } +// resolveCDNChannelManifest resolves the accessory-VFX model list anonymously +// from the asset CDN: channel pointer -> tag -> per-tag vfxs.yml. It ports the +// fail policy of the deleted resolve-accessory-vfx.sh: unreachable inputs fail +// OPEN (errAutogenManifestUnavailable -> optional consumer preserves lock IDs); +// a published-but-broken release (vfxs.yml 404 while haks.json present) or a +// malformed vfxs.yml HARD fail. +func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsumerConfig, src project.AutogenSourceConfig, progress func(string)) (*autogenManifest, error) { + if progress == nil { + progress = func(string) {} + } + + // 1. Offline / air-gapped override: a vfxs.yml path or a manifest-repo + // checkout root containing assets/vfxs.yml. Skips the network entirely. + if envName := strings.TrimSpace(src.OfflineOverrideEnv); envName != "" { + if override := strings.TrimSpace(os.Getenv(envName)); override != "" { + vfxsPath := override + if info, err := os.Stat(override); err == nil && info.IsDir() { + vfxsPath = filepath.Join(override, "assets", "vfxs.yml") + } + raw, err := os.ReadFile(vfxsPath) + if err != nil { + return nil, fmt.Errorf("offline vfxs override %s: %w", vfxsPath, err) + } + entries, err := filterCDNChannelEntries(raw, consumer) + if err != nil { + return nil, err + } + progress(fmt.Sprintf("Using offline vfxs override for %s from %s...", consumer.ID, vfxsPath)) + return &autogenManifest{ID: consumer.Producer, Ref: "local", Entries: entries}, nil + } + } + + // 2. Channel: explicit env, else derived from the git tag. + channel := strings.TrimSpace(os.Getenv(strings.TrimSpace(src.ChannelEnv))) + if channel == "" { + channel = deriveAssetChannel(p.Root) + } + + // 3. CDN base: env override, then config literal, then baked default. + cdnBase := defaultBunnyCDNBase + if envName := strings.TrimSpace(src.CDNBaseEnv); envName != "" { + if v := strings.TrimSpace(os.Getenv(envName)); v != "" { + cdnBase = v + } + } + if v := strings.TrimSpace(src.CDNBase); v != "" { + cdnBase = v + } + cdnBase = strings.TrimRight(cdnBase, "/") + + join := func(path, tag string) string { + return cdnBase + "/" + strings.TrimLeft(strings.ReplaceAll(path, "{tag}", tag), "/") + } + + // 4. channels.json -> tag. Unreachable / non-JSON / channel-absent = fail open. + channelsURL := join(src.ChannelsPath, "") + status, body, err := httpGetStatus(channelsURL) + if err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json unreachable %s: %w", channelsURL, err)) + } + if status != http.StatusOK { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json HTTP %d %s", status, channelsURL)) + } + var channels map[string]string + if err := json.Unmarshal(body, &channels); err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json not JSON %s: %w", channelsURL, err)) + } + tag := strings.TrimSpace(channels[channel]) + if tag == "" { + return nil, unavailableAutogenManifest(fmt.Errorf("channel %q absent in channels.json (%s)", channel, channelsURL)) + } + progress(fmt.Sprintf("Accessory VFX: channel %s -> tag %s", channel, tag)) + + // 5. vfxs.yml for the tag. + manifestURL := join(src.ManifestPath, tag) + vstatus, vbody, err := httpGetStatus(manifestURL) + if err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml unreachable %s: %w", manifestURL, err)) + } + switch vstatus { + case http.StatusOK: + // fall through to parse + case http.StatusNotFound: + // 404 + release marker present = published-but-broken release: HARD fail + // (this is the v0.1.4 silent-drop bug). 404 + marker absent = no published + // release for this tag: fail open. + if marker := strings.TrimSpace(src.ReleaseMarkerPath); marker != "" { + markerURL := join(marker, tag) + if mstatus, _, merr := httpGetStatus(markerURL); merr == nil && mstatus == http.StatusOK { + return nil, fmt.Errorf("release %s has %s but no vfxs.yml (%s) — accessory rows would silently vanish; backfill/republish vfxs.yml for %s", tag, marker, manifestURL, tag) + } + } + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml 404 %s (no published release for %s)", manifestURL, tag)) + default: + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml HTTP %d %s", vstatus, manifestURL)) + } + + entries, err := filterCDNChannelEntries(vbody, consumer) + if err != nil { + return nil, err // malformed vfxs.yml = HARD fail + } + progress(fmt.Sprintf("Accessory VFX: resolved %d model entries from %s", len(entries), manifestURL)) + return &autogenManifest{ID: consumer.Producer, Ref: tag, Entries: entries}, nil +} + +// filterCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under +// vfxs// for the consumer's 4 accessory groups, stripping the leading +// vfxs/ from each source path. A present-but-malformed manifest (unparseable, or +// no assets array) is an error; an empty assets array yields zero entries. +func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) { + var manifest struct { + Assets *[]struct { + Path string `yaml:"path"` + Restype string `yaml:"restype"` + } `yaml:"assets"` + } + if err := yaml.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("malformed vfxs.yml (cannot parse): %w", err) + } + if manifest.Assets == nil { + return nil, fmt.Errorf("malformed vfxs.yml (no assets array)") + } + groups := make(map[string]struct{}, len(consumer.AccessoryVisualeffects.Groups)) + for g := range consumer.AccessoryVisualeffects.Groups { + if g = strings.TrimSpace(g); g != "" { + groups[g] = struct{}{} + } + } + var entries []autogenManifestEntry + for _, a := range *manifest.Assets { + if a.Restype != "mdl" { + continue + } + path := filepath.ToSlash(strings.TrimSpace(a.Path)) + if !strings.HasPrefix(path, "vfxs/") { + continue + } + rel := strings.TrimPrefix(path, "vfxs/") + parts := strings.Split(rel, "/") + if len(parts) < 2 { + continue + } + if _, ok := groups[parts[0]]; !ok { + continue + } + entry := autogenManifestEntry{ + Source: rel, + Group: parts[0], + ModelStem: strings.TrimSuffix(parts[len(parts)-1], ".mdl"), + } + if len(parts) > 2 { + entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") + } + entries = append(entries, entry) + } + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + return strings.Compare(a.Source, b.Source) + }) + return entries, nil +} + +// deriveAssetChannel maps the current git tag (or GITHUB_REF_NAME) to a channel: +// a prerelease tag (v*-*) -> testing; a stable tag (v*) -> current; anything +// else (branch/PR/dev) -> current. Mirrors resolve-accessory-vfx.sh. +func deriveAssetChannel(root string) string { + ref := strings.TrimSpace(os.Getenv("GITHUB_REF_NAME")) + if ref == "" { + if out, err := gitOutput(root, "describe", "--tags", "--exact-match"); err == nil { + ref = strings.TrimSpace(out) + } + } + if strings.HasPrefix(ref, "v") && strings.Contains(ref, "-") { + return "testing" + } + return "current" +} + func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) { root := strings.TrimSpace(consumer.LocalOverrideRoot) if root != "" { diff --git a/internal/topdata/autogen_cdn_channel_test.go b/internal/topdata/autogen_cdn_channel_test.go new file mode 100644 index 0000000..3454150 --- /dev/null +++ b/internal/topdata/autogen_cdn_channel_test.go @@ -0,0 +1,187 @@ +package topdata + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func cdnConsumer() project.AutogenConsumerConfig { + return project.AutogenConsumerConfig{ + ID: "accessory_visualeffects", Producer: "accessory_visualeffects", + Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true, + AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{ + Groups: map[string]project.AccessoryVisualeffectGroupConfig{ + "head_accessories": {}, "chest_accessories": {}, + "head_decorations": {}, "head_features": {}, + }, + }, + Source: project.AutogenSourceConfig{ + Kind: "cdn_channel", + ChannelsPath: "releases/haks/channels.json", + ManifestPath: "releases/haks/{tag}/vfxs.yml", + ReleaseMarkerPath: "releases/haks/{tag}/haks.json", + }, + } +} + +// cdnServer serves channels.json, vfxs.yml, and haks.json from in-memory maps. +// A key absent from the map returns 404. +func cdnServer(t *testing.T, files map[string]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, body) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestResolveCDNChannelHappyPath(t *testing.T) { + root := testProjectRoot(t) + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1.2.3"}`, + "/releases/haks/v1.2.3/vfxs.yml": `assets: + - path: vfxs/head_accessories/hat/hfx_bandana.mdl + restype: mdl + sha256: aaa + - path: vfxs/chest_accessories/cape/cfx_cloak.mdl + restype: mdl + sha256: bbb + - path: vfxs/head_accessories/hat/hfx_bandana.tga + restype: tga + sha256: ccc + - path: vfxs/weapons/sword.mdl + restype: mdl + sha256: ddd +`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + p := testProject(root) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + + m, err := resolveCDNChannelManifest(p, c, c.Source, nil) + if err != nil { + t.Fatalf("resolve failed: %v", err) + } + if len(m.Entries) != 2 { + t.Fatalf("want 2 mdl entries under target groups, got %d: %#v", len(m.Entries), m.Entries) + } + // Sorted by source: chest_accessories/... before head_accessories/... + if m.Entries[0].Source != "chest_accessories/cape/cfx_cloak.mdl" { + t.Fatalf("unexpected first entry: %#v", m.Entries[0]) + } + if m.Entries[1].Source != "head_accessories/hat/hfx_bandana.mdl" || + m.Entries[1].Group != "head_accessories" || + m.Entries[1].Subgroup != "hat" || + m.Entries[1].ModelStem != "hfx_bandana" { + t.Fatalf("unexpected derived entry: %#v", m.Entries[1]) + } + if m.Ref != "v1.2.3" { + t.Fatalf("want ref v1.2.3, got %q", m.Ref) + } +} + +func TestResolveCDNChannelFailOpen(t *testing.T) { + root := testProjectRoot(t) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + t.Run("channels.json unreachable", func(t *testing.T) { + t.Setenv("BUNNY_CDN_BASE", "http://127.0.0.1:0") // unroutable + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) + + t.Run("channel absent", func(t *testing.T) { + srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"testing":"v9"}`}) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) + + t.Run("vfxs 404 no release marker", func(t *testing.T) { + srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"current":"v1"}`}) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) +} + +func TestResolveCDNChannelHardFail(t *testing.T) { + root := testProjectRoot(t) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + t.Run("broken release: vfxs 404 + haks.json present", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/haks.json": `{}`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail, got %v", err) + } + }) + + t.Run("malformed vfxs.yml: no assets array", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/vfxs.yml": "generated_at: 2026-01-01\n", + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail, got %v", err) + } + }) +} + +func TestResolveCDNChannelOfflineOverride(t *testing.T) { + root := testProjectRoot(t) + vfxs := filepath.Join(root, "vfxs.yml") + writeFile(t, vfxs, `assets: + - path: vfxs/head_features/scar/hfx_scar.mdl + restype: mdl + sha256: zzz +`) + t.Setenv("SOW_VFXS_MANIFEST", vfxs) + c := cdnConsumer() + c.Source.OfflineOverrideEnv = "SOW_VFXS_MANIFEST" + + m, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err != nil { + t.Fatalf("offline override failed: %v", err) + } + if len(m.Entries) != 1 || m.Entries[0].Source != "head_features/scar/hfx_scar.mdl" { + t.Fatalf("unexpected offline entries: %#v", m.Entries) + } +} + +func errorIsUnavailable(err error) bool { + return errors.Is(err, errAutogenManifestUnavailable) +} diff --git a/internal/topdata/parts_manifest.go b/internal/topdata/parts_manifest.go index 725a95a..a5e11e0 100644 --- a/internal/topdata/parts_manifest.go +++ b/internal/topdata/parts_manifest.go @@ -274,6 +274,19 @@ func writePartsManifestCache(path string, manifest *partsManifest) error { return nil } +// httpGetStatus performs an anonymous GET and returns the status code plus the +// body (capped). No auth header: cdn_channel inputs (channels.json, vfxs.yml, +// haks.json) are public CDN text — R1. +func httpGetStatus(target string) (int, []byte, error) { + resp, err := partsManifestHTTPClient.Get(target) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + return resp.StatusCode, body, nil +} + func inventoryFromPartsManifest(manifest *partsManifest) map[string]map[int]struct{} { result := make(map[string]map[int]struct{}, len(supportedPartCategories)) for _, category := range supportedPartCategories {