From 3315f8b7eb9e24f24be7008dbd0f2206770d7bf7 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sun, 21 Jun 2026 13:17:58 +0000 Subject: [PATCH] crucible build parity (#14) forces build parity with crucible = local builds and CI/CD builds use different tools. Reviewed-on: https://git.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/14 Co-authored-by: vickydotbat Co-committed-by: vickydotbat --- README.md | 2 +- docs/consumer-contract.md | 41 +- .../plans/2026-06-21-crucible-build-parity.md | 1726 +++++++++++++++++ ...2026-06-21-crucible-build-parity-design.md | 357 ++++ internal/app/app.go | 39 +- internal/dispatch/dispatch.go | 4 +- internal/project/project.go | 44 +- internal/project/project_test.go | 70 + internal/topdata/autogen.go | 264 ++- internal/topdata/autogen_cdn_channel_test.go | 199 ++ .../topdata/autogen_lock_identity_test.go | 114 ++ .../topdata/autogen_manifest_file_test.go | 2 +- internal/topdata/lfs_materialize_test.go | 29 + internal/topdata/parity_guard_test.go | 56 + internal/topdata/parts_manifest.go | 13 + internal/topdata/top_package.go | 83 + internal/topdata/topdata_test.go | 6 +- 17 files changed, 2996 insertions(+), 53 deletions(-) create mode 100644 docs/superpowers/plans/2026-06-21-crucible-build-parity.md create mode 100644 docs/superpowers/specs/2026-06-21-crucible-build-parity-design.md create mode 100644 internal/topdata/autogen_cdn_channel_test.go create mode 100644 internal/topdata/autogen_lock_identity_test.go create mode 100644 internal/topdata/lfs_materialize_test.go create mode 100644 internal/topdata/parity_guard_test.go diff --git a/README.md b/README.md index f231eba..6010a82 100644 --- a/README.md +++ b/README.md @@ -97,5 +97,5 @@ publish event is a `v*` tag (see `sow-docs/runbooks/ci-trigger-standard.md`). ## Consumers -How the artifact repos resolve a Crucible binary (and the `NWN_ROOT` rule) is +How the artifact repos resolve a Crucible binary is documented in [`docs/consumer-contract.md`](docs/consumer-contract.md). diff --git a/docs/consumer-contract.md b/docs/consumer-contract.md index dc6f114..491213d 100644 --- a/docs/consumer-contract.md +++ b/docs/consumer-contract.md @@ -22,30 +22,37 @@ If none resolve, the wrapper exits non-zero with a Phase 5 message. The wrappers prefer the single-token `crucible-` shim so `"$builder" args` quoting stays correct. -## In CI (preferred) +## Parity contract — wrappers are not authoritative for build inputs -**OPERATOR NOTE: Stop. Do not do it this way.** -**Use the pinned `prod.yml` version. That's what it's there for.** +A consumer build wrapper may only: -~~Run the consumer job inside the pinned image and the binaries are on `PATH`:~~ +1. **validate** — a pre-build gate that does not change output; +2. **publish** — a post-build step on a separate artifact; +3. **resolve the crucible binary** — the bootstrap above. -```yaml -container: - image: registry.westgate.pw/deployment/crucible: # pinned, immutable -``` +A wrapper MUST NOT fetch, resolve, generate, or stage any input Crucible reads to +produce the artifact. If Crucible needs an input, Crucible acquires it. This makes +the "no local-machine assumptions" + "same inputs → identical output" rules +explicit: local `crucible ` and CI `crucible ` against the same +committed config must produce identical bytes, with no CI-only pre-steps. -~~No host install, no `$HOME` layout, no developer machine assumptions.~~ +Builders read every input from the project tree. No NWN install is required. A +builder that one day needs NWN game data auto-detects the install (the NWN home +and Steam path are reliably discoverable), with an optional explicit override for +non-standard layouts — never a required hand-set env. -## `NWN_ROOT` rule +## In CI -Crucible **never guesses `NWN_ROOT` from `$HOME`** (this was a deploy-notes -pain point). The NWN install/data root is passed explicitly: +Builds run the Crucible **binary** — from the `crucible.sh` / `crucible.ps1` +bootstrap (anonymous download) or, for Nix users, from the flake devshell pinned +by `flake.lock`. CI runs the _same_ `crucible ` as local dev, inside the +nix devshell (binary-cache fast). No build container, no token, no CI-only +pre-steps. -- env `NWN_ROOT=/path/to/nwn`, or -- flag `--nwn-root /path/to/nwn`. - -A builder that needs `NWN_ROOT` and receives neither must fail closed with a -clear message, not fall back to a home-directory default. +The `registry.westgate.pw/deployment/crucible:` image is **deployment-only**: +`prod.yml` pins it so tools travel with the runtime host (`nwn.enable`), a +disabled placeholder until the NWN stack lands. It is **not** a build tool and no +build/CI job consumes it. ## Determinism diff --git a/docs/superpowers/plans/2026-06-21-crucible-build-parity.md b/docs/superpowers/plans/2026-06-21-crucible-build-parity.md new file mode 100644 index 0000000..922e310 --- /dev/null +++ b/docs/superpowers/plans/2026-06-21-crucible-build-parity.md @@ -0,0 +1,1726 @@ +# Crucible Build Parity Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `crucible topdata build-topdata` reproduce the CI build byte-for-byte from a bare clone — moving accessory-VFX resolution and Git-LFS materialization out of the wrapper and into Crucible, anchoring autogen lock IDs to the model path, and converging every consumer on one anonymously-fetched, flake-pinned Crucible. + +**Architecture:** Crucible (Go, `sow-tools`) becomes the single authority for every build input. A new `cdn_channel` autogen source resolves `channels.json` → tag → `vfxs.yml` over the anonymous CDN inside Go (replacing `scripts/resolve-accessory-vfx.sh`). Crucible materializes Git-LFS asset bytes before packing the hak. Autogen lock identity moves from a config-derived presentation key to a model-path key so config edits no longer reshuffle IDs. Consumer flakes pull `crucible` from a `sow-tools` flake input; the bash wrappers thin to validate→resolve→build. + +**Tech Stack:** Go 1.x (stdlib + `gopkg.in/yaml.v3`), Nix flakes, Gitea Actions, bash. + +## Global Constraints + +- **R1 — No extra secrets.** All build-input reads anonymous; `CRUCIBLE_TOKEN` removed from consumer CI. LFS uses the clone's own credentials. +- **R2 — No hand-written config.** Every build default baked into committed `nwn-tool.yaml` + Crucible defaults. `crucible topdata build-topdata` with no flags/env must work. +- **R3 — No host data assumptions, no `NWN_ROOT`.** Inputs come from the project tree only. No NWN install required. +- **R4 — Nix is convenience, not a requirement.** Non-Nix (incl. Windows) builds via the `crucible.sh`/`crucible.ps1` bootstrap + host `git-lfs`; same build path. +- **R5 — Same Crucible, both worlds.** Local and CI run the identical `crucible topdata build-topdata` against the same committed config. No CI-only pre-steps that change output. +- **Pin source of truth = `flake.lock` per repo.** Reject floating "latest" for parity-critical builds. +- **Determinism scope:** same commit + same channel state → identical bytes. A `v*` ref / explicit `SOW_TOPDATA_ASSET_CHANNEL` is fully pinned. +- Go module path prefix: `git.westgate.pw/ShadowsOverWestgate/sow-tools`. +- CDN default base: `https://depot.westgate.pw`. +- The 4 accessory groups: `chest_accessories`, `head_accessories`, `head_decorations`, `head_features`. + +--- + +## Phase 1 — sow-tools (Crucible) Go changes + +> All Phase 1 tasks run from `sow-tools/`. Test command throughout: `go test ./internal/...`. Build check: `make build`. + +### Task 1: `AutogenSourceConfig` struct, field, and validation + +**Files:** + +- Modify: `sow-tools/internal/project/project.go` (add struct near line 414 `AutogenDeriveConfig`; add field to `AutogenConsumerConfig` at line 355-369; add validation) +- Test: `sow-tools/internal/project/project_test.go` + +**Interfaces:** + +- Produces: + - `type AutogenSourceConfig struct{ Kind, CDNBase, CDNBaseEnv, ChannelsPath, ManifestPath, ReleaseMarkerPath, ChannelEnv, OfflineOverrideEnv string }` + - `AutogenConsumerConfig.Source AutogenSourceConfig` (yaml/json key `source`) + - `func validateAutogenConsumerSources(consumers []AutogenConsumerConfig) []string` + +- [ ] **Step 1: Write the failing validation test** + +In `sow-tools/internal/project/project_test.go` add: + +```go +func TestValidateAutogenConsumerSourcesCDNChannel(t *testing.T) { + cases := []struct { + name string + source AutogenSourceConfig + wantFail bool + }{ + {"valid", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json", ManifestPath: "releases/haks/{tag}/vfxs.yml"}, false}, + {"missing channels_path", AutogenSourceConfig{Kind: "cdn_channel", ManifestPath: "releases/haks/{tag}/vfxs.yml"}, true}, + {"missing manifest_path", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json"}, true}, + {"manifest_path lacks {tag}", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json", ManifestPath: "releases/haks/vfxs.yml"}, true}, + {"empty kind is ignored", AutogenSourceConfig{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + failures := validateAutogenConsumerSources([]AutogenConsumerConfig{{ID: "x", Source: tc.source}}) + if tc.wantFail && len(failures) == 0 { + t.Fatalf("expected validation failure, got none") + } + if !tc.wantFail && len(failures) != 0 { + t.Fatalf("expected no failure, got %v", failures) + } + }) + } +} +``` + +- [ ] **Step 2: Run it to confirm it fails to compile** + +Run: `go test ./internal/project/ -run TestValidateAutogenConsumerSourcesCDNChannel` +Expected: FAIL — `undefined: validateAutogenConsumerSources` / `AutogenSourceConfig`. + +- [ ] **Step 3: Add the struct and field** + +In `sow-tools/internal/project/project.go`, add to `AutogenConsumerConfig` (after the `ManifestFile` field at line 368): + +```go + Source AutogenSourceConfig `json:"source,omitempty" yaml:"source,omitempty"` +``` + +Add the struct (place it next to `AutogenDeriveConfig`, around line 417): + +```go +type AutogenSourceConfig struct { + Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` + CDNBase string `json:"cdn_base,omitempty" yaml:"cdn_base,omitempty"` + CDNBaseEnv string `json:"cdn_base_env,omitempty" yaml:"cdn_base_env,omitempty"` + ChannelsPath string `json:"channels_path,omitempty" yaml:"channels_path,omitempty"` + ManifestPath string `json:"manifest_path,omitempty" yaml:"manifest_path,omitempty"` + ReleaseMarkerPath string `json:"release_marker_path,omitempty" yaml:"release_marker_path,omitempty"` + ChannelEnv string `json:"channel_env,omitempty" yaml:"channel_env,omitempty"` + OfflineOverrideEnv string `json:"offline_override_env,omitempty" yaml:"offline_override_env,omitempty"` +} +``` + +- [ ] **Step 4: Add the validation function** + +In `sow-tools/internal/project/project.go`, add: + +```go +func validateAutogenConsumerSources(consumers []AutogenConsumerConfig) []string { + var failures []string + for _, c := range consumers { + if strings.TrimSpace(c.Source.Kind) != "cdn_channel" { + continue + } + label := strings.TrimSpace(c.ID) + if label == "" { + label = "" + } + if strings.TrimSpace(c.Source.ChannelsPath) == "" { + failures = append(failures, fmt.Sprintf("autogen consumer %s: source.channels_path is required for kind cdn_channel", label)) + } + manifestPath := strings.TrimSpace(c.Source.ManifestPath) + if manifestPath == "" { + failures = append(failures, fmt.Sprintf("autogen consumer %s: source.manifest_path is required for kind cdn_channel", label)) + } else if !strings.Contains(manifestPath, "{tag}") { + failures = append(failures, fmt.Sprintf("autogen consumer %s: source.manifest_path must contain {tag}", label)) + } + } + return failures +} +``` + +(`fmt` and `strings` are already imported in `project.go`.) + +- [ ] **Step 5: Wire validation into `Validate`** + +Find the block in `project.go` that appends topdata validation failures (around line 639, the `failures = append(failures, validateTopData...)` cluster). Add immediately after it: + +```go + failures = append(failures, validateAutogenConsumerSources(effective.Autogen.Consumers)...) +``` + +If `effective` does not expose `Autogen`, use the same receiver the surrounding lines use for autogen (grep `effective.Autogen` / `p.Config.Autogen` in `Validate` and match the existing convention). + +- [ ] **Step 6: Run the test to confirm it passes** + +Run: `go test ./internal/project/ -run TestValidateAutogenConsumerSourcesCDNChannel` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add internal/project/project.go internal/project/project_test.go +git commit -m "feat(project): add cdn_channel AutogenSourceConfig + validation" +``` + +--- + +### Task 2: `cdn_channel` manifest resolution + +**Files:** + +- Modify: `sow-tools/internal/topdata/autogen.go` (new resolution branch + helpers; imports) +- Modify: `sow-tools/internal/topdata/parts_manifest.go` (add `httpGetStatus`) +- Create: `sow-tools/internal/topdata/autogen_cdn_channel_test.go` + +**Interfaces:** + +- Consumes: `project.AutogenConsumerConfig.Source` (Task 1); `autogenManifest`, `autogenManifestEntry`, `unavailableAutogenManifest`, `errAutogenManifestUnavailable`, `gitOutput`, `partsManifestHTTPClient` (existing). +- Produces: + - `func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsumerConfig, src project.AutogenSourceConfig, progress func(string)) (*autogenManifest, error)` + - `func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error)` + - `func deriveAssetChannel(root string) string` + - `func httpGetStatus(target string) (int, []byte, error)` (in `parts_manifest.go`) + - `const defaultBunnyCDNBase = "https://depot.westgate.pw"` + +- [ ] **Step 1: Write the failing resolution test** + +Create `sow-tools/internal/topdata/autogen_cdn_channel_test.go`: + +```go +package topdata + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func cdnConsumer() project.AutogenConsumerConfig { + return project.AutogenConsumerConfig{ + ID: "accessory_visualeffects", Producer: "accessory_visualeffects", + Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true, + AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{ + Groups: map[string]project.AccessoryVisualeffectGroupConfig{ + "head_accessories": {}, "chest_accessories": {}, + "head_decorations": {}, "head_features": {}, + }, + }, + Source: project.AutogenSourceConfig{ + Kind: "cdn_channel", + ChannelsPath: "releases/haks/channels.json", + ManifestPath: "releases/haks/{tag}/vfxs.yml", + ReleaseMarkerPath: "releases/haks/{tag}/haks.json", + }, + } +} + +// cdnServer serves channels.json, vfxs.yml, and haks.json from in-memory maps. +// A key absent from the map returns 404. +func cdnServer(t *testing.T, files map[string]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, body) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestResolveCDNChannelHappyPath(t *testing.T) { + root := testProjectRoot(t) + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1.2.3"}`, + "/releases/haks/v1.2.3/vfxs.yml": `assets: + - path: vfxs/head_accessories/hat/hfx_bandana.mdl + restype: mdl + sha256: aaa + - path: vfxs/chest_accessories/cape/cfx_cloak.mdl + restype: mdl + sha256: bbb + - path: vfxs/head_accessories/hat/hfx_bandana.tga + restype: tga + sha256: ccc + - path: vfxs/weapons/sword.mdl + restype: mdl + sha256: ddd +`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + p := testProject(root) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + + m, err := resolveCDNChannelManifest(p, c, c.Source, nil) + if err != nil { + t.Fatalf("resolve failed: %v", err) + } + if len(m.Entries) != 2 { + t.Fatalf("want 2 mdl entries under target groups, got %d: %#v", len(m.Entries), m.Entries) + } + // Sorted by source: chest_accessories/... before head_accessories/... + if m.Entries[0].Source != "chest_accessories/cape/cfx_cloak.mdl" { + t.Fatalf("unexpected first entry: %#v", m.Entries[0]) + } + if m.Entries[1].Source != "head_accessories/hat/hfx_bandana.mdl" || + m.Entries[1].Group != "head_accessories" || + m.Entries[1].Subgroup != "hat" || + m.Entries[1].ModelStem != "hfx_bandana" { + t.Fatalf("unexpected derived entry: %#v", m.Entries[1]) + } + if m.Ref != "v1.2.3" { + t.Fatalf("want ref v1.2.3, got %q", m.Ref) + } +} +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `go test ./internal/topdata/ -run TestResolveCDNChannelHappyPath` +Expected: FAIL — `undefined: resolveCDNChannelManifest`. + +- [ ] **Step 3: Add `httpGetStatus` to `parts_manifest.go`** + +In `sow-tools/internal/topdata/parts_manifest.go` (it already imports `io` and `net/http`), add: + +```go +// httpGetStatus performs an anonymous GET and returns the status code plus the +// body (capped). No auth header: cdn_channel inputs (channels.json, vfxs.yml, +// haks.json) are public CDN text — R1. +func httpGetStatus(target string) (int, []byte, error) { + resp, err := partsManifestHTTPClient.Get(target) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + return resp.StatusCode, body, nil +} +``` + +- [ ] **Step 4: Add the resolution branch and helpers to `autogen.go`** + +In `sow-tools/internal/topdata/autogen.go`, add `net/http` and `gopkg.in/yaml.v3` to the import block. + +Add the const near the top (after line 18 `autogenManifestCacheMaxAge`): + +```go +const defaultBunnyCDNBase = "https://depot.westgate.pw" +``` + +In `resolveAutogenConsumerManifest` (line 227), add the new branch as the first check: + +```go +func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) { + if strings.TrimSpace(consumer.Source.Kind) == "cdn_channel" { + return resolveCDNChannelManifest(p, consumer, consumer.Source, progress) + } + if manifestFile := strings.TrimSpace(consumer.ManifestFile); manifestFile != "" { + return readAutogenConsumerManifestFile(p, consumer, manifestFile, progress) + } + // ... rest unchanged +``` + +Add the helpers (anywhere in `autogen.go`, e.g. after `readAutogenConsumerManifestFile`): + +```go +// resolveCDNChannelManifest resolves the accessory-VFX model list anonymously +// from the asset CDN: channel pointer -> tag -> per-tag vfxs.yml. It ports the +// fail policy of the deleted resolve-accessory-vfx.sh: unreachable inputs fail +// OPEN (errAutogenManifestUnavailable -> optional consumer preserves lock IDs); +// a published-but-broken release (vfxs.yml 404 while haks.json present) or a +// malformed vfxs.yml HARD fail. +func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsumerConfig, src project.AutogenSourceConfig, progress func(string)) (*autogenManifest, error) { + if progress == nil { + progress = func(string) {} + } + + // 1. Offline / air-gapped override: a vfxs.yml path or a manifest-repo + // checkout root containing assets/vfxs.yml. Skips the network entirely. + if envName := strings.TrimSpace(src.OfflineOverrideEnv); envName != "" { + if override := strings.TrimSpace(os.Getenv(envName)); override != "" { + vfxsPath := override + if info, err := os.Stat(override); err == nil && info.IsDir() { + vfxsPath = filepath.Join(override, "assets", "vfxs.yml") + } + raw, err := os.ReadFile(vfxsPath) + if err != nil { + return nil, fmt.Errorf("offline vfxs override %s: %w", vfxsPath, err) + } + entries, err := filterCDNChannelEntries(raw, consumer) + if err != nil { + return nil, err + } + progress(fmt.Sprintf("Using offline vfxs override for %s from %s...", consumer.ID, vfxsPath)) + return &autogenManifest{ID: consumer.Producer, Ref: "local", Entries: entries}, nil + } + } + + // 2. Channel: explicit env, else derived from the git tag. + channel := strings.TrimSpace(os.Getenv(strings.TrimSpace(src.ChannelEnv))) + if channel == "" { + channel = deriveAssetChannel(p.Root) + } + + // 3. CDN base: env override, then config literal, then baked default. + cdnBase := defaultBunnyCDNBase + if envName := strings.TrimSpace(src.CDNBaseEnv); envName != "" { + if v := strings.TrimSpace(os.Getenv(envName)); v != "" { + cdnBase = v + } + } + if v := strings.TrimSpace(src.CDNBase); v != "" { + cdnBase = v + } + cdnBase = strings.TrimRight(cdnBase, "/") + + join := func(path, tag string) string { + return cdnBase + "/" + strings.TrimLeft(strings.ReplaceAll(path, "{tag}", tag), "/") + } + + // 4. channels.json -> tag. Unreachable / non-JSON / channel-absent = fail open. + channelsURL := join(src.ChannelsPath, "") + status, body, err := httpGetStatus(channelsURL) + if err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json unreachable %s: %w", channelsURL, err)) + } + if status != http.StatusOK { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json HTTP %d %s", status, channelsURL)) + } + var channels map[string]string + if err := json.Unmarshal(body, &channels); err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json not JSON %s: %w", channelsURL, err)) + } + tag := strings.TrimSpace(channels[channel]) + if tag == "" { + return nil, unavailableAutogenManifest(fmt.Errorf("channel %q absent in channels.json (%s)", channel, channelsURL)) + } + progress(fmt.Sprintf("Accessory VFX: channel %s -> tag %s", channel, tag)) + + // 5. vfxs.yml for the tag. + manifestURL := join(src.ManifestPath, tag) + vstatus, vbody, err := httpGetStatus(manifestURL) + if err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml unreachable %s: %w", manifestURL, err)) + } + switch vstatus { + case http.StatusOK: + // fall through to parse + case http.StatusNotFound: + // 404 + release marker present = published-but-broken release: HARD fail + // (this is the v0.1.4 silent-drop bug). 404 + marker absent = no published + // release for this tag: fail open. + if marker := strings.TrimSpace(src.ReleaseMarkerPath); marker != "" { + markerURL := join(marker, tag) + if mstatus, _, merr := httpGetStatus(markerURL); merr == nil && mstatus == http.StatusOK { + return nil, fmt.Errorf("release %s has %s but no vfxs.yml (%s) — accessory rows would silently vanish; backfill/republish vfxs.yml for %s", tag, marker, manifestURL, tag) + } + } + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml 404 %s (no published release for %s)", manifestURL, tag)) + default: + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml HTTP %d %s", vstatus, manifestURL)) + } + + entries, err := filterCDNChannelEntries(vbody, consumer) + if err != nil { + return nil, err // malformed vfxs.yml = HARD fail + } + progress(fmt.Sprintf("Accessory VFX: resolved %d model entries from %s", len(entries), manifestURL)) + return &autogenManifest{ID: consumer.Producer, Ref: tag, Entries: entries}, nil +} + +// filterCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under +// vfxs// for the consumer's 4 accessory groups, stripping the leading +// vfxs/ from each source path. A present-but-malformed manifest (unparseable, or +// no assets array) is an error; an empty assets array yields zero entries. +func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) { + var manifest struct { + Assets *[]struct { + Path string `yaml:"path"` + Restype string `yaml:"restype"` + } `yaml:"assets"` + } + if err := yaml.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("malformed vfxs.yml (cannot parse): %w", err) + } + if manifest.Assets == nil { + return nil, fmt.Errorf("malformed vfxs.yml (no assets array)") + } + groups := make(map[string]struct{}, len(consumer.AccessoryVisualeffects.Groups)) + for g := range consumer.AccessoryVisualeffects.Groups { + if g = strings.TrimSpace(g); g != "" { + groups[g] = struct{}{} + } + } + var entries []autogenManifestEntry + for _, a := range *manifest.Assets { + if a.Restype != "mdl" { + continue + } + path := filepath.ToSlash(strings.TrimSpace(a.Path)) + if !strings.HasPrefix(path, "vfxs/") { + continue + } + rel := strings.TrimPrefix(path, "vfxs/") + parts := strings.Split(rel, "/") + if len(parts) < 2 { + continue + } + if _, ok := groups[parts[0]]; !ok { + continue + } + entry := autogenManifestEntry{ + Source: rel, + Group: parts[0], + ModelStem: strings.TrimSuffix(parts[len(parts)-1], ".mdl"), + } + if len(parts) > 2 { + entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") + } + entries = append(entries, entry) + } + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + return strings.Compare(a.Source, b.Source) + }) + return entries, nil +} + +// deriveAssetChannel maps the current git tag (or GITHUB_REF_NAME) to a channel: +// a prerelease tag (v*-*) -> testing; a stable tag (v*) -> current; anything +// else (branch/PR/dev) -> current. Mirrors resolve-accessory-vfx.sh. +func deriveAssetChannel(root string) string { + ref := strings.TrimSpace(os.Getenv("GITHUB_REF_NAME")) + if ref == "" { + if out, err := gitOutput(root, "describe", "--tags", "--exact-match"); err == nil { + ref = strings.TrimSpace(out) + } + } + if strings.HasPrefix(ref, "v") && strings.Contains(ref, "-") { + return "testing" + } + return "current" +} +``` + +- [ ] **Step 5: Run the happy-path test** + +Run: `go test ./internal/topdata/ -run TestResolveCDNChannelHappyPath` +Expected: PASS. + +- [ ] **Step 6: Add the fail-policy tests** + +Append to `autogen_cdn_channel_test.go`: + +```go +func TestResolveCDNChannelFailOpen(t *testing.T) { + root := testProjectRoot(t) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + t.Run("channels.json unreachable", func(t *testing.T) { + t.Setenv("BUNNY_CDN_BASE", "http://127.0.0.1:0") // unroutable + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) + + t.Run("channel absent", func(t *testing.T) { + srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"testing":"v9"}`}) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) + + t.Run("vfxs 404 no release marker", func(t *testing.T) { + srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"current":"v1"}`}) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) +} + +func TestResolveCDNChannelHardFail(t *testing.T) { + root := testProjectRoot(t) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + t.Run("broken release: vfxs 404 + haks.json present", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/haks.json": `{}`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail, got %v", err) + } + }) + + t.Run("malformed vfxs.yml: no assets array", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/vfxs.yml": "generated_at: 2026-01-01\n", + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail, got %v", err) + } + }) +} + +func errorIsUnavailable(err error) bool { + return errors.Is(err, errAutogenManifestUnavailable) +} +``` + +Add `"errors"` to the test file's import block. + +- [ ] **Step 7: Run all cdn_channel tests** + +Run: `go test ./internal/topdata/ -run 'TestResolveCDNChannel'` +Expected: PASS (all subtests). + +- [ ] **Step 8: Add the offline-override test** + +Append to `autogen_cdn_channel_test.go`: + +```go +func TestResolveCDNChannelOfflineOverride(t *testing.T) { + root := testProjectRoot(t) + vfxs := filepath.Join(root, "vfxs.yml") + writeFile(t, vfxs, `assets: + - path: vfxs/head_features/scar/hfx_scar.mdl + restype: mdl + sha256: zzz +`) + t.Setenv("SOW_VFXS_MANIFEST", vfxs) + c := cdnConsumer() + c.Source.OfflineOverrideEnv = "SOW_VFXS_MANIFEST" + + m, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err != nil { + t.Fatalf("offline override failed: %v", err) + } + if len(m.Entries) != 1 || m.Entries[0].Source != "head_features/scar/hfx_scar.mdl" { + t.Fatalf("unexpected offline entries: %#v", m.Entries) + } +} +``` + +Add `"path/filepath"` to the test imports. + +- [ ] **Step 9: Run the full topdata suite** + +Run: `go test ./internal/topdata/` +Expected: PASS. Then `make build` — Expected: builds clean. + +- [ ] **Step 10: Commit** + +```bash +git add internal/topdata/autogen.go internal/topdata/parts_manifest.go internal/topdata/autogen_cdn_channel_test.go +git commit -m "feat(topdata): resolve accessory VFX from cdn_channel inside Crucible" +``` + +--- + +### Task 3: Model-anchored autogen lock identity + cutover remap + stale pruning + +**Files:** + +- Modify: `sow-tools/internal/topdata/autogen.go` (`augmentWithAutogeneratedAccessoryVisualeffects` at line 819; `autogenConsumerManagedLockKeyMatcher` at line 178; add `accessoryVisualeffectLockKey`) +- Create: `sow-tools/internal/topdata/autogen_lock_identity_test.go` + +**Interfaces:** + +- Consumes: `autogenManifestEntry`, `accessoryVisualeffectIdentity`, `resolveAccessoryVisualeffectsPolicy` (existing). +- Produces: `func accessoryVisualeffectLockKey(dataset string, entry autogenManifestEntry) string`. Lock key form: `{dataset}:{model_source_path}`, e.g. `visualeffects:head_accessories/hat/hfx_bandana.mdl`. + +- [ ] **Step 1: Write the failing config-stability test** + +Create `sow-tools/internal/topdata/autogen_lock_identity_test.go`: + +```go +package topdata + +import ( + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func accConsumer() project.AutogenConsumerConfig { + return project.AutogenConsumerConfig{ + ID: "accessory_visualeffects", Producer: "accessory_visualeffects", + Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true, + AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{ + Groups: map[string]project.AccessoryVisualeffectGroupConfig{ + "head_accessories": {ModelColumns: []string{"Imp_HeadCon_Node"}}, + }, + GroupTokenSource: "folder_name", CategoryFrom: "immediate_parent", + Delimiter: "/", Case: "preserve", + StripModelPrefixes: []string{"hfx_"}, + KeyFormat: "{dataset}:{group}{delimiter}{category_segment}{stem}", + LabelFormat: "{group}{delimiter}{category_segment}{stem}", + ModelColumn: "Imp_HeadCon_Node", + }, + } +} + +func vfxEntry() autogenManifestEntry { + return autogenManifestEntry{ + Source: "head_accessories/hat/hfx_bandana.mdl", Group: "head_accessories", + Subgroup: "hat", ModelStem: "hfx_bandana", + } +} + +func vfxDataset(lock map[string]int) nativeCollectedDataset { + return nativeCollectedDataset{ + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Imp_HeadCon_Node"}, + Rows: nil, + LockData: lock, + } +} + +func TestAccessoryLockKeyIsModelAnchored(t *testing.T) { + got := accessoryVisualeffectLockKey("visualeffects", vfxEntry()) + want := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + if got != want { + t.Fatalf("lock key = %q, want %q", got, want) + } +} + +// A config-only change (here: a different key_format) must NOT move the ID, +// because identity is the model path, not the presentation key. +func TestConfigOnlyChangeKeepsLockID(t *testing.T) { + lockKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{lockKey: 555})} + + c := accConsumer() + c.AccessoryVisualeffects.KeyFormat = "{dataset}:{stem}" // config-only change + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, c, nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if got[0].LockData[lockKey] != 555 { + t.Fatalf("expected ID preserved at 555, got %#v", got[0].LockData) + } +} +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `go test ./internal/topdata/ -run 'TestAccessoryLockKey|TestConfigOnlyChange'` +Expected: FAIL — `undefined: accessoryVisualeffectLockKey` and ID not preserved. + +- [ ] **Step 3: Add the lock-key helper** + +In `autogen.go` add: + +```go +// accessoryVisualeffectLockKey is the model-anchored autogen lock identity: +// the dataset namespace prefix plus the model source path from vfxs.yml (leading +// vfxs/ already stripped on entry.Source). Config-derived presentation parts +// (group token, category, delimiter, case, prefix-stripping) are deliberately +// excluded so a config-only edit never reshuffles row IDs. +func accessoryVisualeffectLockKey(dataset string, entry autogenManifestEntry) string { + return dataset + ":" + strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(entry.Source)), "vfxs/") +} +``` + +- [ ] **Step 4: Rework the augmentor to key the lock on the model path** + +In `augmentWithAutogeneratedAccessoryVisualeffects` (line 819), replace the per-entry loop (lines 873-895, from `for _, entry := range entries {` through the closing brace before the `slices.SortFunc`) with: + +```go + liveLockKeys := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + key, label, modelStem, groupPolicy, ok := accessoryVisualeffectIdentity(dataset.Dataset.Name, entry, policy) + if !ok { + continue + } + lockKey := accessoryVisualeffectLockKey(dataset.Dataset.Name, entry) + liveLockKeys[lockKey] = struct{}{} + + if existing, exists := rowByKey[key]; exists { + applyDiscoveredAccessoryVisualeffectDefaults(existing, dataset.Columns, modelStem, label, policy, groupPolicy) + if _, pinned := lockData[lockKey]; !pinned { + if rowID, ok := existing["id"].(int); ok { + lockData[lockKey] = rowID + } + } + continue + } + + rowID, ok := lockData[lockKey] + if !ok { + switch { + case historicalLockData[lockKey] != 0 || hasKey(historicalLockData, lockKey): + rowID = historicalLockData[lockKey] + case hasKey(historicalLockData, key): + // One-time cutover: adopt the ID from the old config-derived key + // so IDs carry over with zero shift on the key-scheme switch. The + // stale old key is pruned below. + rowID = historicalLockData[key] + default: + rowID = allocateNextID() + } + lockData[lockKey] = rowID + } + newRow := createDefaultAccessoryVisualeffectRow(dataset.Columns, rowID, key, label, modelStem, policy, groupPolicy) + rows = append(rows, newRow) + rowByKey[key] = newRow + } + + // Stale pruning: drop managed lock keys (this consumer's accessory keys, + // incl. now-superseded old config-scheme keys) that no live entry resolves + // to, freeing their IDs for first-free reuse. Safe here because the + // augmentor only runs on a successful, non-empty resolution; the fail-open + // paths skip it entirely and never reach this code. + if managed := autogenConsumerManagedLockKeyMatcher(consumer); managed != nil { + for lk := range lockData { + if _, live := liveLockKeys[lk]; live { + continue + } + if managed(lk) { + delete(lockData, lk) + } + } + } +``` + +Add the small helper (the `int`-zero ambiguity guard) to `autogen.go`: + +```go +func hasKey(m map[string]int, key string) bool { + _, ok := m[key] + return ok +} +``` + +(Then simplify the first `case` to `case hasKey(historicalLockData, lockKey):`.) + +- [ ] **Step 5: Fix the case expression** + +Replace the awkward first case with the clean form: + +```go + switch { + case hasKey(historicalLockData, lockKey): + rowID = historicalLockData[lockKey] + case hasKey(historicalLockData, key): + rowID = historicalLockData[key] + default: + rowID = allocateNextID() + } +``` + +- [ ] **Step 6: Make the managed-key matcher model-anchored** + +Replace the body of `autogenConsumerManagedLockKeyMatcher` (line 178) for the `accessory_visualeffects` case with a config-independent matcher (dataset prefix + first path segment is one of the groups). This matches both old config-scheme keys and new model-path keys, so fail-open preservation and stale pruning both stay correct: + +```go +func autogenConsumerManagedLockKeyMatcher(consumer project.AutogenConsumerConfig) func(string) bool { + switch consumer.Mode { + case "accessory_visualeffects": + dataset := strings.TrimSpace(consumer.Dataset) + if dataset == "" { + dataset = "visualeffects" + } + prefix := dataset + ":" + groups := make(map[string]struct{}, len(consumer.AccessoryVisualeffects.Groups)) + for group := range consumer.AccessoryVisualeffects.Groups { + if group = strings.TrimSpace(group); group != "" { + groups[group] = struct{}{} + } + } + if len(groups) == 0 { + return nil + } + return func(key string) bool { + if !strings.HasPrefix(key, prefix) { + return false + } + rest := key[len(prefix):] + idx := strings.Index(rest, "/") + if idx <= 0 { + return false + } + _, ok := groups[rest[:idx]] + return ok + } + default: + return nil + } +} +``` + +- [ ] **Step 7: Run the config-stability tests** + +Run: `go test ./internal/topdata/ -run 'TestAccessoryLockKey|TestConfigOnlyChange'` +Expected: PASS. + +- [ ] **Step 8: Add re-export, rename/remove, cutover, and fail-open tests** + +Append to `autogen_lock_identity_test.go`: + +```go +// A model re-export (same path) keeps its ID — content change != identity change. +func TestModelReExportKeepsLockID(t *testing.T) { + lockKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{lockKey: 777})} + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if got[0].LockData[lockKey] != 777 { + t.Fatalf("expected ID 777 preserved, got %#v", got[0].LockData) + } +} + +// A removed model frees its old ID; the surviving model keeps its lock entry. +func TestModelRemovalPrunesStaleLockKey(t *testing.T) { + keep := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + gone := "visualeffects:head_accessories/hat/hfx_removed.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{keep: 1, gone: 2})} + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if _, present := got[0].LockData[gone]; present { + t.Fatalf("expected removed model's lock key pruned, got %#v", got[0].LockData) + } + if got[0].LockData[keep] != 1 { + t.Fatalf("expected surviving model ID 1 kept, got %#v", got[0].LockData) + } +} + +// Cutover: a lock that only has the OLD config-derived key adopts that ID under +// the new model-path key, with zero shift, and prunes the old key. +func TestCutoverRemapPreservesID(t *testing.T) { + oldKey := "visualeffects:head_accessories/hat/bandana" // {dataset}:{group}/{category}/{stem-without-hfx_} + newKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{oldKey: 4242})} + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if got[0].LockData[newKey] != 4242 { + t.Fatalf("expected cutover to carry ID 4242 to new key, got %#v", got[0].LockData) + } + if _, present := got[0].LockData[oldKey]; present { + t.Fatalf("expected old config key pruned after cutover, got %#v", got[0].LockData) + } +} +``` + +> Note for the implementer: the old config key in `TestCutoverRemapPreservesID` must equal what `accessoryVisualeffectIdentity` produces for `vfxEntry()` under `accConsumer()`'s policy. If the suite reports a different computed key, set `oldKey` to the value the test prints — do not change the production code to match a guessed key. + +- [ ] **Step 9: Run the full identity suite + regression suite** + +Run: `go test ./internal/topdata/ -run 'TestAccessory|TestConfigOnly|TestModel|TestCutover'` +Expected: PASS. +Run: `go test ./internal/topdata/` +Expected: PASS — confirms `autogen_preserve_test.go` (fail-open preservation) still holds with the new model-anchored matcher. + +- [ ] **Step 10: Commit** + +```bash +git add internal/topdata/autogen.go internal/topdata/autogen_lock_identity_test.go +git commit -m "feat(topdata): anchor autogen lock identity on model path with cutover remap" +``` + +--- + +### Task 4: Git-LFS materialization inside Crucible + +**Files:** + +- Modify: `sow-tools/internal/topdata/top_package.go` (`packageBuiltTopData` at line 122; add `materializeAssetLFS`, `hasLFSPointerStubs`; add `bytes` import) +- Create: `sow-tools/internal/topdata/lfs_materialize_test.go` + +**Interfaces:** + +- Consumes: `gitOutput` (assets_repo.go), `project.Project.Root`, `project.Project.TopDataSourceDir()`. +- Produces: + - `func materializeAssetLFS(p *project.Project, progress func(string)) error` + - `func hasLFSPointerStubs(root string) bool` + - Skip control: env `CRUCIBLE_SKIP_LFS` (read inside `materializeAssetLFS`). + +- [ ] **Step 1: Write the failing stub-detection test** + +Create `sow-tools/internal/topdata/lfs_materialize_test.go`: + +```go +package topdata + +import ( + "path/filepath" + "testing" +) + +func TestHasLFSPointerStubsDetectsStub(t *testing.T) { + root := t.TempDir() + stub := "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 12345\n" + writeFile(t, filepath.Join(root, "a", "model.mdl"), stub) + writeFile(t, filepath.Join(root, "a", "real.txt"), "not a pointer, real bytes here") + + if !hasLFSPointerStubs(root) { + t.Fatal("expected stub detection to be true") + } +} + +func TestHasLFSPointerStubsNoStub(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "real.mdl"), "binary-ish content without a pointer header") + if hasLFSPointerStubs(root) { + t.Fatal("expected no stub detected") + } +} +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `go test ./internal/topdata/ -run TestHasLFSPointerStubs` +Expected: FAIL — `undefined: hasLFSPointerStubs`. + +- [ ] **Step 3: Add the materialization helpers** + +In `sow-tools/internal/topdata/top_package.go` add `"bytes"` to the import block, then add: + +```go +// materializeAssetLFS ensures the topdata asset tree holds real bytes, not +// Git-LFS pointer stubs, before the hak is packed. A bare CI checkout does not +// smudge LFS, so a direct Crucible build would otherwise ship ~130-byte stubs. +// It uses the clone's own credentials (R1) and hard-fails if stubs survive the +// pull. Set CRUCIBLE_SKIP_LFS=1 to opt out (maintain-tree regenerates the data/ +// tree and discards the package, so it must not hard-depend on LFS). +func materializeAssetLFS(p *project.Project, progress func(string)) error { + if progress == nil { + progress = func(string) {} + } + assetsDir := filepath.Join(p.TopDataSourceDir(), "assets") + info, err := os.Stat(assetsDir) + if err != nil || !info.IsDir() { + return nil // no asset tree to materialize + } + if !hasLFSPointerStubs(assetsDir) { + return nil // already real bytes (or repo does not use LFS) + } + if isTruthyEnv(os.Getenv("CRUCIBLE_SKIP_LFS")) { + progress("CRUCIBLE_SKIP_LFS set: leaving Git-LFS pointer stubs unmaterialized (package will be discarded)") + return nil + } + if _, err := gitOutput(p.Root, "lfs", "version"); err != nil { + return fmt.Errorf("asset tree has Git-LFS pointer stubs but git-lfs is unavailable; install git-lfs or enter the nix devshell: %w", err) + } + progress("Materializing Git-LFS assets (git lfs pull)...") + // A fresh clone may lack the lfs smudge/clean filters; install them locally + // first (scoped to this repo's .git/config, no host mutation). Idempotent. + if _, err := gitOutput(p.Root, "lfs", "install", "--local"); err != nil { + return fmt.Errorf("git lfs install --local failed: %w", err) + } + if _, err := gitOutput(p.Root, "lfs", "pull"); err != nil { + return fmt.Errorf("git lfs pull failed: %w", err) + } + if hasLFSPointerStubs(assetsDir) { + return fmt.Errorf("refusing to build from pointer stubs: Git-LFS pointer stubs remain under %s after git lfs pull", assetsDir) + } + return nil +} + +// hasLFSPointerStubs reports whether any small text file under root is a Git-LFS +// pointer. Real assets are larger than any pointer stub, so the size gate keeps +// this cheap on a tree of binaries. +func hasLFSPointerStubs(root string) bool { + found := false + _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || found { + return nil + } + info, ierr := d.Info() + if ierr != nil || info.Size() == 0 || info.Size() > 1024 { + return nil + } + f, oerr := os.Open(path) + if oerr != nil { + return nil + } + defer f.Close() + buf := make([]byte, 256) + n, _ := f.Read(buf) + if bytes.Contains(buf[:n], []byte("git-lfs.github.com/spec/v1")) { + found = true + } + return nil + }) + return found +} + +func isTruthyEnv(v string) bool { + switch strings.TrimSpace(strings.ToLower(v)) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} +``` + +- [ ] **Step 4: Call materialization before packing** + +In `packageBuiltTopData` (line 122), immediately after the `if progress == nil` guard and before `outputHAK := p.TopDataPackageHAKPath()`, add: + +```go + if err := materializeAssetLFS(p, progress); err != nil { + return PackageResult{}, err + } +``` + +- [ ] **Step 5: Run the stub-detection tests** + +Run: `go test ./internal/topdata/ -run TestHasLFSPointerStubs` +Expected: PASS. + +- [ ] **Step 6: Run the full suite + build** + +Run: `go test ./internal/topdata/` then `make build` +Expected: PASS, clean build. + +- [ ] **Step 7: Commit** + +```bash +git add internal/topdata/top_package.go internal/topdata/lfs_materialize_test.go +git commit -m "feat(topdata): materialize Git-LFS asset bytes inside Crucible before packing" +``` + +~~> Skipped: a `--skip-lfs` CLI flag. The `CRUCIBLE_SKIP_LFS` env covers the only opt-out consumer (maintain-tree CI); add the flag if a human ever needs it interactively.~~ + +> Operator redaction: Please add a `--skip-lfs` CLI flag in addition to the `CRUCIBLE_SKIP_LFS` env. + +--- + +### Task 5: NWN_ROOT cleanup + +**Files:** + +- Modify: `sow-tools/internal/dispatch/dispatch.go:251-254` +- Modify: `sow-tools/docs/consumer-contract.md` (remove `## NWN_ROOT rule`, lines 39-48) +- Modify: `sow-tools/README.md:100` +- Modify: `sow-tools/internal/dispatch/dispatch_test.go` (if it asserts the old help text) + +**Interfaces:** none (docs + help text only). + +- [ ] **Step 1: Check whether a test pins the old help text** + +Run: `grep -n "NWN_ROOT\|never guesses" internal/dispatch/dispatch_test.go` +If it asserts the old "passed explicitly via NWN_ROOT" string, update that expectation in the same step you change the help text below. + +- [ ] **Step 2: Replace the dispatch help text** + +In `sow-tools/internal/dispatch/dispatch.go`, replace lines 251-254 (the three `NWN_ROOT` Fprintf lines) with: + +```go + fmt.Fprintf(w, "\nBuilders read all inputs from the project tree (nwn-tool.yaml + data/); no\n") + fmt.Fprintf(w, "NWN install is required. A builder that one day needs NWN game data\n") + fmt.Fprintf(w, "auto-detects the install; it never requires a hand-set path. See\n") + fmt.Fprintf(w, "docs/consumer-contract.md.\n") +``` + +- [ ] **Step 3: Remove the `NWN_ROOT rule` section from the contract doc** + +In `sow-tools/docs/consumer-contract.md`, delete the entire `## `NWN_ROOT` rule` section (the heading through the "fall back to a home-directory default." paragraph). (The replacement auto-detect rule lands in Task 6's contract rewrite.) + +- [ ] **Step 4: Fix the README pointer** + +In `sow-tools/README.md:100`, change the line that reads +`How the artifact repos resolve a Crucible binary (and the `NWN_ROOT` rule) is` +to drop the `NWN_ROOT` clause: +`How the artifact repos resolve a Crucible binary is` +(Verify the surrounding sentence still reads correctly; adjust the trailing words if needed.) + +- [ ] **Step 5: Verify build + dispatch test** + +Run: `go test ./internal/dispatch/` then `make build` +Expected: PASS, clean build. Then `grep -rn "NWN_ROOT" internal/ README.md docs/consumer-contract.md` — Expected: no matches (the only remaining mention is `docs/migration-from-nwn-tool.md`, left as historical record). + +- [ ] **Step 6: Commit** + +```bash +git add internal/dispatch/dispatch.go internal/dispatch/dispatch_test.go docs/consumer-contract.md README.md +git commit -m "chore(crucible): drop dead NWN_ROOT rule; auto-detect is the forward rule" +``` + +--- + +### Task 6: Consumer-contract parity rules + image deployment-only + +**Files:** + +- Modify: `sow-tools/docs/consumer-contract.md` + +**Interfaces:** none (doc only). This is the written half of the parity guarantee; Task 7 is the enforced half. + +- [ ] **Step 1: Add the parity contract section** + +In `sow-tools/docs/consumer-contract.md`, add a new section (place it after the resolution-order section): + +```markdown +## Parity contract — wrappers are not authoritative for build inputs + +A consumer build wrapper may only: + +1. **validate** — a pre-build gate that does not change output; +2. **publish** — a post-build step on a separate artifact; +3. **resolve the crucible binary** — the bootstrap above. + +A wrapper MUST NOT fetch, resolve, generate, or stage any input Crucible reads to +produce the artifact. If Crucible needs an input, Crucible acquires it. This makes +the "no local-machine assumptions" + "same inputs → identical output" rules +explicit: local `crucible ` and CI `crucible ` against the same +committed config must produce identical bytes, with no CI-only pre-steps. + +Builders read every input from the project tree. No NWN install is required. A +builder that one day needs NWN game data auto-detects the install (the NWN home +and Steam path are reliably discoverable), with an optional explicit override for +non-standard layouts — never a required hand-set env. +``` + +- [ ] **Step 2: Replace the CI image guidance with deployment-only** + +In `consumer-contract.md`, replace the entire `## In CI (preferred)` section (the OPERATOR NOTE + struck-through `container: image:` guidance) with: + +```markdown +## In CI + +Builds run the Crucible **binary** — from the `crucible.sh` / `crucible.ps1` +bootstrap (anonymous download) or, for Nix users, from the flake devshell pinned +by `flake.lock`. CI runs the _same_ `crucible ` as local dev, inside the +nix devshell (binary-cache fast). No build container, no token, no CI-only +pre-steps. + +The `registry.westgate.pw/deployment/crucible:` image is **deployment-only**: +`prod.yml` pins it so tools travel with the runtime host (`nwn.enable`), a +disabled placeholder until the NWN stack lands. It is **not** a build tool and no +build/CI job consumes it. +``` + +- [ ] **Step 3: Verify** + +Run: `grep -n "Stop. Do not\|container:\|pinned image" docs/consumer-contract.md` +Expected: no matches. + +- [ ] **Step 4: Commit** + +```bash +git add docs/consumer-contract.md +git commit -m "docs(contract): add wrapper parity rules; state crucible image is deployment-only" +``` + +--- + +### Task 7: CI parity guard (hermetic integration test) + +**Files:** + +- Create: `sow-tools/internal/topdata/parity_guard_test.go` + +**Interfaces:** + +- Consumes: `resolveCDNChannelManifest` (Task 2), `augmentWithAutogeneratedAccessoryVisualeffects` (Task 3), `applyAutogenConsumers` (existing). + +This is the test that would have caught the v0.1.4 / local-zero-rows bug: with **no `manifest_file`, no `NWN_ROOT`, no token**, a `cdn_channel` consumer must contribute accessory rows. It drives the full `applyAutogenConsumers` path against an httptest CDN, mechanically enforcing R2/R3/R5 for the resolution+augmentation pipeline. (The operational, whole-build guard lands as a CI job in Task 11.) + +- [ ] **Step 1: Write the parity guard test** + +Create `sow-tools/internal/topdata/parity_guard_test.go`: + +```go +package topdata + +import ( + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +// PARITY GUARD: a bare project (no manifest_file, no NWN_ROOT, no token) with a +// cdn_channel consumer must produce accessory rows. If accessory resolution ever +// drifts back out of Crucible into a wrapper, this fails. +func TestParityGuardCDNChannelProducesAccessoryRows(t *testing.T) { + root := testProjectRoot(t) + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1.0.0"}`, + "/releases/haks/v1.0.0/vfxs.yml": `assets: + - path: vfxs/head_accessories/hat/hfx_bandana.mdl + restype: mdl + sha256: aaa + - path: vfxs/chest_accessories/cape/cfx_cloak.mdl + restype: mdl + sha256: bbb +`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + // Prove R3: no NWN_ROOT in the environment. + t.Setenv("NWN_ROOT", "") + + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + c.AccessoryVisualeffects.ModelColumn = "Imp_HeadCon_Node" + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{c} + + collected := []nativeCollectedDataset{{ + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Imp_HeadCon_Node", "Imp_Root_S_Node"}, + Rows: nil, + LockData: map[string]int{}, + }} + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got[0].Rows) != 2 { + t.Fatalf("PARITY GUARD: expected 2 accessory rows, got %d (%#v)", len(got[0].Rows), got[0].Rows) + } + wantKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + if _, ok := got[0].LockData[wantKey]; !ok { + t.Fatalf("PARITY GUARD: expected lock id for %s, got %#v", wantKey, got[0].LockData) + } +} +``` + +- [ ] **Step 2: Run the parity guard** + +Run: `go test ./internal/topdata/ -run TestParityGuard` +Expected: PASS. + +- [ ] **Step 3: Run the whole sow-tools suite** + +Run: `make check` +Expected: PASS (go vet + go test + shellcheck + yamllint). + +- [ ] **Step 4: Commit** + +```bash +git add internal/topdata/parity_guard_test.go +git commit -m "test(topdata): parity guard — cdn_channel yields accessory rows with no wrapper/env" +``` + +--- + +## Phase 2 — Release Crucible + +> Operator step, no code. Flag down the operator and remind them. After Phase 1 merges, cut a Crucible `v*` release so the bootstrap and flake input resolve the parity build. Assets are already anon-downloadable (R1, verified in the design). Record the released version (e.g. `v0.3.0`) — Phase 3 pins it via `flake.lock`. + +- [ ] **Step 1:** Merge Phase 1; tag/release Crucible per the repo's existing release workflow. Note the resulting version tag for Phase 3. + +--- + +## Phase 3 — Toolchain convergence (flakes) + +> One binary, one pin per repo, three modes — all anonymous, all identical. sow-tools' flake already exposes `packages..crucible`; consumers add it as a flake input. + +### Task 8: sow-topdata flake — add the crucible input + devshell + +**Files:** + +- Modify: `sow-topdata/flake.nix` +- Regenerate: `sow-topdata/flake.lock` + +**Interfaces:** + +- Consumes: `sow-tools` flake output `packages.${system}.crucible` (exists in `sow-tools/flake.nix`). + +- [ ] **Step 1: Add the sow-tools input and put crucible on PATH** + +Edit `sow-topdata/flake.nix`. Add the input and thread it through: + +```nix +{ + description = "sow-topdata — 2DA/TLK/wiki validate/build dev shell"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.sow-tools.url = "git+https://git.westgate.pw/ShadowsOverWestgate/sow-tools"; + inputs.sow-tools.inputs.nixpkgs.follows = "nixpkgs"; + + outputs = { self, nixpkgs, sow-tools }: + let + systems = [ "x86_64-linux" "aarch64-linux" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system}); + in + { + devShells = forAllSystems (system: pkgs: { + default = pkgs.mkShell { + name = "sow-topdata"; + packages = with pkgs; [ + bashInteractive + git + git-lfs + shellcheck + yamllint + jq + yq-go + zstd + prettier + ] ++ [ sow-tools.packages.${system}.crucible ]; + shellHook = '' + echo "sow-topdata devshell. crucible, git-lfs, shellcheck, yamllint, jq, yq, zstd, prettier ready." + echo " scripts/validate-topdata.sh structural checks" + echo " scripts/build-topdata.sh build package via crucible" + echo " scripts/format-tree.sh reformat tree to .editorconfig" + ''; + }; + }); + }; +} +``` + +> Note: `forAllSystems` now passes both `system` and `pkgs`. If the repo's existing helper signature differs, match the convention used by `sow-assets-manifest/flake.nix` (which already pins crucible per-system) rather than introducing a new pattern. + +- [ ] **Step 2: Lock the input** + +Run: `cd sow-topdata && nix flake update sow-tools` (or `nix flake lock --update-input sow-tools`) +Expected: `flake.lock` gains a `sow-tools` node pinned to the Phase 2 release commit. + +- [ ] **Step 3: Verify crucible is on PATH in the devshell** + +Run: `cd sow-topdata && nix develop --command crucible version` +Expected: prints the pinned Crucible version (the Phase 2 release). + +- [ ] **Step 4: Commit** + +```bash +git add sow-topdata/flake.nix sow-topdata/flake.lock +git commit -m "build(topdata): provide crucible from sow-tools flake input in the devshell" +``` + +--- + +### Task 9: sow-assets-manifest flake — switch off the manual fetchurl pin + +**Files:** + +- Modify: `sow-assets-manifest/flake.nix` +- Regenerate: `sow-assets-manifest/flake.lock` + +**Interfaces:** + +- Consumes: `sow-tools` flake output `packages.${system}.crucible`. + +- [ ] **Step 1: Replace the `fetchurl`/hash crucible derivation with the flake input** + +Edit `sow-assets-manifest/flake.nix`. Add the input: + +```nix + inputs.sow-tools.url = "git+https://git.westgate.pw/ShadowsOverWestgate/sow-tools"; + inputs.sow-tools.inputs.nixpkgs.follows = "nixpkgs"; +``` + +Change the outputs signature to `{ self, nixpkgs, sow-tools }`, delete the entire `crucibleVersion` / `crucibleArch` / `crucibleHashes` / `crucible = pkgs.stdenvNoCC.mkDerivation { ... }` block, and replace the `++ [ crucible ]` line in `packages` with: + +```nix + ++ [ sow-tools.packages.${pkgs.system}.crucible ]; +``` + +(`forAllSystems` here already passes `pkgs`; `pkgs.system` is available.) + +- [ ] **Step 2: Lock the input** + +Run: `cd sow-assets-manifest && nix flake lock --update-input sow-tools` +Expected: `flake.lock` gains the `sow-tools` node. + +- [ ] **Step 3: Verify** + +Run: `cd sow-assets-manifest && nix develop --command crucible version` +Expected: same pinned version as Task 8. + +- [ ] **Step 4: Commit** + +```bash +git add sow-assets-manifest/flake.nix sow-assets-manifest/flake.lock +git commit -m "build(assets-manifest): pin crucible via sow-tools flake input, drop manual fetchurl hash" +``` + +--- + +## Phase 4 — sow-topdata config, wrapper, CI + +### Task 10: Swap `manifest_file:` → `source:` in nwn-tool.yaml + +**Files:** + +- Modify: `sow-topdata/nwn-tool.yaml:99-140` + +**Interfaces:** + +- Consumes: the `cdn_channel` source schema (Task 1) and resolution (Task 2). + +- [ ] **Step 1: Replace the consumer source declaration** + +In `sow-topdata/nwn-tool.yaml`, in the `autogen.consumers` accessory entry, delete line 106 (`manifest_file: .cache/sow-accessory-vfx-manifest.json`) and insert the `source:` block between `optional: true` (line 105) and `accessory_visualeffects:` (line 107): + +```yaml +optional: true +source: + kind: cdn_channel + cdn_base_env: BUNNY_CDN_BASE + channels_path: releases/haks/channels.json + manifest_path: releases/haks/{tag}/vfxs.yml + release_marker_path: releases/haks/{tag}/haks.json + channel_env: SOW_TOPDATA_ASSET_CHANNEL + offline_override_env: SOW_VFXS_MANIFEST +accessory_visualeffects: +``` + +Leave the entire `accessory_visualeffects:` policy block (groups, group_token_source, delimiter, key_format, etc.) unchanged. Update the explanatory comment at lines 11-16 to describe Crucible's `cdn_channel` resolution instead of `scripts/resolve-accessory-vfx.sh`. + +- [ ] **Step 2: Validate the config** + +Run: `cd sow-topdata && nix develop --command crucible topdata config --validate` (use the actual validate subcommand; if unsure run `crucible topdata --help`). +Expected: no validation errors about `autogen.consumers[].source`. + +- [ ] **Step 3: Commit** + +```bash +git add sow-topdata/nwn-tool.yaml +git commit -m "config(topdata): resolve accessory VFX via cdn_channel source (was manifest_file)" +``` + +--- + +### Task 11: Delete the resolver script, thin the wrapper, add the CI parity job + +**Files:** + +- Delete: `sow-topdata/scripts/resolve-accessory-vfx.sh` +- Delete: `sow-topdata/tests/resolve-accessory-vfx-contract.sh` +- Modify: `sow-topdata/scripts/build-topdata.sh` +- Modify: `sow-topdata/scripts/lib.sh` (remove `materialize_lfs_assets`) +- Create: `sow-topdata/.gitea/workflows/parity-guard.yml` + +**Interfaces:** + +- Consumes: Crucible now owns VFX resolution (Task 2/10) and LFS (Task 4). + +- [ ] **Step 1: Delete the resolver script and its bash contract test** + +Run: + +```bash +cd sow-topdata +git rm scripts/resolve-accessory-vfx.sh tests/resolve-accessory-vfx-contract.sh +``` + +(If `tests/resolve-accessory-vfx-contract.sh` is referenced by a test runner or Makefile, remove that reference too — grep first: `grep -rn "resolve-accessory-vfx" .`) + +- [ ] **Step 2: Thin `build-topdata.sh`** + +Replace `sow-topdata/scripts/build-topdata.sh` with the validate→resolve→build shape (no resolver pre-step, no LFS pull — Crucible owns both now): + +```bash +#!/usr/bin/env bash +# Build the topdata package (.cache/sow_top.hak + .cache/sow_tlk.tlk) via Crucible. +# Crucible reads ./nwn-tool.yaml, resolves accessory VFX from the asset CDN, +# materializes Git-LFS asset bytes, compiles data/ into 2da + the custom TLK, and +# writes the package into paths.build (.cache). This wrapper only: validate (gate) +# -> resolve crucible -> build. It is NOT authoritative for any build input +# (see sow-tools/docs/consumer-contract.md). FAILS CLOSED if crucible cannot be +# resolved — it never fakes a package. +# shellcheck source=lib.sh +. "$(dirname "$0")/lib.sh" + +cd "$REPO_ROOT" + +"$REPO_ROOT/scripts/validate-topdata.sh" + +if ! crucible="$(resolve_crucible)"; then + die "crucible not found. Install it on PATH, set \$CRUCIBLE, enter the nix \ +devshell, or keep ./crucible.sh (the bootstrap wrapper) in the repo." +fi + +log "building topdata package via $crucible" +mkdir -p .cache +"$crucible" topdata build-topdata + +hak=".cache/sow_top.hak" +tlk=".cache/sow_tlk.tlk" +[[ -f "$hak" && -f "$tlk" ]] || die "builder did not produce the top package ($hak + $tlk)" +log "build-topdata: OK -> $hak + $tlk" +``` + +- [ ] **Step 3: Remove `materialize_lfs_assets` from `lib.sh`** + +In `sow-topdata/scripts/lib.sh`, delete the `materialize_lfs_assets()` function (Crucible owns LFS now). Keep `log/warn/die/need/git_sha/resolve_crucible`. Run `grep -rn "materialize_lfs_assets" sow-topdata/` afterward — Expected: no remaining callers. + +- [ ] **Step 4: Add the operational parity-guard CI job** + +Create `sow-topdata/.gitea/workflows/parity-guard.yml` — runs Crucible **directly** (no wrapper, no token, no `NWN_ROOT`) in a bare checkout and asserts the package builds with accessory rows: + +```yaml +# Operational parity guard: build the topdata package with Crucible ALONE — no +# build-topdata.sh, no CRUCIBLE_TOKEN, no NWN_ROOT, anonymous CDN — and assert +# the built visualeffects.2da carries accessory rows. This is the enforced half +# of the parity contract (sow-tools/docs/consumer-contract.md): it fails if any +# build-output-affecting step ever drifts back out of Crucible into a wrapper. +name: parity-guard + +on: + pull_request: + +jobs: + parity: + runs-on: nix-docker + steps: + - uses: actions/checkout@v4 + with: + lfs: false # prove Crucible materializes LFS itself + + - name: Build with crucible alone (no wrapper, no env) + run: nix develop --command crucible topdata build-topdata + + - name: Assert accessory rows present + run: | + nix develop --command bash -euo pipefail -c ' + crucible topdata convert 2da-to-json .cache/visualeffects.2da .cache/visualeffects.json || \ + crucible-hak extract .cache/sow_top.hak visualeffects.2da .cache/visualeffects.2da + grep -Eq "head_accessories|chest_accessories|head_decorations|head_features" .cache/visualeffects.json \ + || { echo "PARITY GUARD: no accessory rows in built visualeffects.2da"; exit 1; } + ' +``` + +> Implementer note: confirm the exact path Crucible writes the compiled `visualeffects.2da` to (it lands under `paths.build` = `.cache`; run `crucible topdata build-topdata` once locally and `find .cache -name 'visualeffects.2da'`). Wire the assert step to that real path; the `convert`/`extract` fallback above is a sketch — replace it with the one command that actually reads the built 2da. The assertion (accessory group tokens present) is the load-bearing part. + +> Implementer note: `visualeffects.2da` should appear at `sow-topdata/.cache/2da/visualeffects.2da` after a `build-topdata` run. + +- [ ] **Step 5: Lint the workflows and scripts** + +Run: `cd sow-topdata && nix develop --command bash -c 'shellcheck scripts/build-topdata.sh scripts/lib.sh && yamllint .gitea/workflows/parity-guard.yml'` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add -A sow-topdata/scripts sow-topdata/tests sow-topdata/.gitea/workflows/parity-guard.yml +git commit -m "build(topdata): thin wrapper (Crucible owns VFX+LFS); add operational parity guard" +``` + +--- + +### Task 12: Drop `CRUCIBLE_TOKEN`, swap the LFS skip env, build inside the devshell + +**Files:** + +- Modify: `sow-topdata/.gitea/workflows/build-topdata.yml` +- Modify: `sow-topdata/.gitea/workflows/maintain-tree.yml` +- Audit: every `sow-topdata/.gitea/workflows/*.yml` for `CRUCIBLE_TOKEN` / `TOPDATA_SKIP_LFS` + +**Interfaces:** + +- Consumes: anonymous Crucible (R1), `CRUCIBLE_SKIP_LFS` (Task 4). + +- [ ] **Step 1: Find every reference** + +Run: `grep -rn "CRUCIBLE_TOKEN\|TOPDATA_SKIP_LFS" sow-topdata/.gitea/workflows/` +Expected references: `build-topdata.yml` (CRUCIBLE_TOKEN) and `maintain-tree.yml` (both). + +- [ ] **Step 2: Remove `CRUCIBLE_TOKEN` from build-topdata.yml** + +In `sow-topdata/.gitea/workflows/build-topdata.yml`, delete the `env:` block carrying `CRUCIBLE_TOKEN` from the "Build topdata package" step (the binary now downloads anonymously / comes from the devshell). Update the leading comment that references `CRUCIBLE_TOKEN` to state the build is anonymous via the nix devshell. The step command stays `nix develop --command scripts/build-topdata.sh`. + +- [ ] **Step 3: Swap the LFS skip env in maintain-tree.yml** + +In `sow-topdata/.gitea/workflows/maintain-tree.yml`, in the "Regenerate lockfiles" step: delete `CRUCIBLE_TOKEN` from `env:`, and replace `TOPDATA_SKIP_LFS: "1"` with `CRUCIBLE_SKIP_LFS: "1"`. Update the adjacent comment to point at Crucible's skip control rather than the wrapper's. Command stays `nix develop --command scripts/build-topdata.sh`. + +- [ ] **Step 4: Sweep the remaining workflows** + +For any other workflow that set `CRUCIBLE_TOKEN` to fetch the binary (e.g. `publish-topdata.yml`, `release.yml`, `drift-check.yml` — check what grep returned), remove the token if the job builds via the nix devshell. Leave tokens that authenticate a _publish_ (release upload), which are unrelated to fetching crucible. + +- [ ] **Step 5: Lint** + +Run: `cd sow-topdata && nix develop --command yamllint .gitea/workflows/` +Expected: clean. Then `grep -rn "CRUCIBLE_TOKEN\|TOPDATA_SKIP_LFS" .gitea/workflows/` — Expected: no build-time CRUCIBLE_TOKEN, no TOPDATA_SKIP_LFS. + +- [ ] **Step 6: Commit** + +```bash +git add sow-topdata/.gitea/workflows +git commit -m "ci(topdata): drop vestigial CRUCIBLE_TOKEN; use CRUCIBLE_SKIP_LFS for maintain-tree" +``` + +--- + +## Phase 5 — Verification + +### Task 13: End-to-end parity verification + +**Files:** none (verification only). + +- [ ] **Step 1: Bare-clone build, no env, no key** + +In a fresh clone of `sow-topdata` (or a clean checkout with cleared `.cache`): + +```bash +rm -rf .cache +nix develop --command crucible topdata build-topdata +``` + +Expected: succeeds; `.cache/sow_top.hak` + `.cache/sow_tlk.tlk` produced. Confirm accessory rows: + +```bash +find .cache -name 'visualeffects.2da' # locate the built 2da +# convert/extract and count rows under the 4 accessory groups +``` + +Expected: accessory rows present (the design's reference figure is 637 rows for the current channel state; the exact count tracks the live channel). + +- [ ] **Step 2: Same command, no wrapper, with LFS unsmudged** + +```bash +git lfs uninstall --local 2>/dev/null || true +rm -rf .cache +nix develop --command crucible topdata build-topdata +``` + +Expected: Crucible materializes LFS itself; the hak holds real bytes (no pointer stubs), same accessory rows. + +- [ ] **Step 3: Lock stability across a config-only edit** + +Temporarily change a presentation-only field in `nwn-tool.yaml` (e.g. `label_format`), rebuild, and `git diff data/visualeffects/lock.json`. +Expected: **no ID changes** (only labels/columns may differ). Revert the edit. + +- [ ] **Step 4: Full check suites** + +```bash +cd sow-tools && make check +cd ../sow-topdata && nix develop --command scripts/validate-topdata.sh +``` + +Expected: both green. + +- [ ] **Step 5: Confirm the parity guard catches drift (sanity)** + +Run `go test ./internal/topdata/ -run TestParityGuard` in `sow-tools`. +Expected: PASS. (Optional: prove it fails if resolution is disabled, then revert.) + +--- + +## Notes / deferred (not this plan) + +- Parts `2da` / `cachedmodels` reinstatement can reuse the `cdn_channel` source type — separate spec. +- Roll the thinned-wrapper pattern review across `sow-module` / `sow-codebase`. +- A future `id`+`sha` object form for the lock is a clean later add (the lock stays a flat `string→int` map here); the audit trail comes free from `git diff` on the committed `vfxs.yml`. diff --git a/docs/superpowers/specs/2026-06-21-crucible-build-parity-design.md b/docs/superpowers/specs/2026-06-21-crucible-build-parity-design.md new file mode 100644 index 0000000..884582d --- /dev/null +++ b/docs/superpowers/specs/2026-06-21-crucible-build-parity-design.md @@ -0,0 +1,357 @@ +# Crucible build parity — local == CI, zero-config reproducible + +Date: 2026-06-21 +Status: approved (design); implementation pending +Repos touched: `sow-tools` (Crucible + contract doc + image notes), +`sow-topdata` (config, flake, wrapper, CI), `sow-assets-manifest` (flake pin), +and the consumer wrapper/flake pattern shared by every Crucible repo. + +Supersedes the resolver-script approach in +`2026-06-21-topdata-vfx-autogen-channel-design.md` (the channel→tag→`vfxs.yml` +resolution moves out of a bash wrapper and into Crucible). + +## Problem + +`crucible topdata build-topdata` run by hand does **not** reproduce the CI +build. CI runs `scripts/build-topdata.sh`, which resolves the accessory-VFX +model list with `scripts/resolve-accessory-vfx.sh` and writes +`.cache/sow-accessory-vfx-manifest.json` **before** Crucible runs; Crucible only +reads that pre-staged file (`nwn-tool.yaml: manifest_file:`). Run Crucible +directly and the file never exists, the `optional` consumer skips, and +`visualeffects.2da` ships **zero accessory rows** — exactly the local build the +operator observed (637 rows in CI, 0 locally). + +Root cause: a **build-output-affecting input is resolved outside Crucible**, in +a wrapper script. The wrapper, not Crucible, is authoritative for that input. + +## Goal + +Crucible is the single authority for everything that lands in a build artifact. +A clone of any consumer repo, on any system, builds the same bytes CI builds — +with no secret beyond what cloning already required and no hand-written +configuration. + +### Zero-config reproducibility requirements + +- **R1 — No extra secrets.** All build-input reads are anonymous: the asset + channel pointer (`releases/haks/channels.json`) and `vfxs.yml` are public CDN + text. The crucible binary is fetchable anonymously — **verified**: the + sow-tools release assets return HTTP 200 without auth, and the bootstrap tries + anon before any token, so `CRUCIBLE_TOKEN` in consumer CI is **vestigial and is + removed**. Git-LFS asset bytes use the **same** credentials the clone already + used — never a separate key. +- **R2 — No hand-written config.** Every default needed to build is baked into + committed `nwn-tool.yaml` and Crucible defaults: CDN base, channel selection, + source paths. The user types `crucible topdata build-topdata` and nothing else. +- **R3 — No host data assumptions, no `NWN_ROOT`.** Builders read every input + from the project tree (`nwn-tool.yaml` + committed `data/**/base.json`); no NWN + install is required. `NWN_ROOT` is **dead** today — nothing in Crucible reads it + (see *NWN_ROOT cleanup* below). The accessory build reads + `data/visualeffects/base.json`. (Enforced by the CI parity guard, which runs in + a bare checkout with no `NWN_ROOT`.) +- **R4 — Nix is convenience, not a requirement.** The flake devshell provides + `crucible`, `git-lfs`, `jq`, `yq`, etc. on `PATH` for Nix users. Non-Nix users + (incl. Windows) get the same Crucible via the `crucible.sh`/`crucible.ps1` + bootstrap and a host `git-lfs`; the build path through Crucible is identical. +- **R5 — Same Crucible, both worlds.** Local dev and CI invoke the same Crucible + subcommand against the same committed config. No CI-only pre-steps that change + output. + +## Parity contract (prevents recurrence) + +Added to `sow-tools/docs/consumer-contract.md`. A consumer build wrapper may only: + +1. **validate** — a pre-build gate that does not change output; +2. **publish** — post-build, on a separate artifact; +3. **resolve the crucible binary** — the bootstrap in `consumer-contract.md`. + +A wrapper MUST NOT fetch, resolve, generate, or stage any input Crucible reads +to produce the artifact. If Crucible needs an input, Crucible acquires it. This +is the existing "no local-machine assumptions" + "same inputs → identical +output" rule, made explicit: **the wrapper is not allowed to be authoritative +for build inputs.** + +## Design + +### A. Accessory-VFX resolution moves into Crucible + +**Config (`sow-topdata/nwn-tool.yaml`).** Drop `manifest_file:`; declare the +source on the consumer. The 4-group + `mdl` filter is **not** re-declared — it +is derived from the keys already present under `accessory_visualeffects.groups`: + +```yaml +autogen: + consumers: + - id: accessory_visualeffects + producer: accessory_visualeffects + dataset: visualeffects + mode: accessory_visualeffects + optional: true + source: + kind: cdn_channel + cdn_base_env: BUNNY_CDN_BASE # default https://depot.westgate.pw + channels_path: releases/haks/channels.json + manifest_path: releases/haks/{tag}/vfxs.yml + release_marker_path: releases/haks/{tag}/haks.json # the broken-release guard probe + channel_env: SOW_TOPDATA_ASSET_CHANNEL # else derive from git tag + offline_override_env: SOW_VFXS_MANIFEST # dev/air-gapped vfxs.yml path + accessory_visualeffects: + groups: { chest_accessories: {...}, head_accessories: {...}, + head_decorations: {...}, head_features: {...} } + # ... rest of the policy block unchanged +``` + +New struct field on `AutogenConsumerConfig` (sow-tools +`internal/project/project.go`): `Source AutogenSourceConfig` with the keys +above. Validation: when `source.kind == cdn_channel`, `channels_path` and +`manifest_path` are required and `manifest_path` must contain `{tag}`. + +**Resolution (`internal/topdata/autogen.go`, +`resolveAutogenConsumerManifest`).** A new branch for `source.kind == +cdn_channel`, reusing the existing `fetchJSON` HTTP client and the project YAML +parser: + +1. **Offline override.** If `offline_override_env` names a readable file (or a + manifest-repo checkout root containing `assets/vfxs.yml`), parse it and skip + the network — the dev/air-gapped path. +2. **Channel.** `channel_env` if set; else from the git tag / `GITHUB_REF_NAME` + (`v*-*` → `testing`, `v*` → `current`, otherwise `current`). +3. **CDN base.** `cdn_base_env` value, else the baked default. +4. `GET {cdn}/{channels_path}` → JSON → `tag = channels[channel]`. +5. `GET {cdn}/{manifest_path with {tag}}` → parse YAML `assets[]`. +6. **Filter** to `restype == mdl` whose `path` is under `vfxs//` for `` in + the `accessory_visualeffects.groups` keys. Derive each entry's `source` + (leading `vfxs/` stripped), `group`, `subgroup`, `model_stem`. Sort by + `source`. Hand the entries to the existing accessory augmentor unchanged. + +**Fail policy (ported verbatim from the resolver — this is the v0.1.4-bug +guard).** Distinguish *unreachable* (fail open) from *broken* (hard fail): + +- channels.json unreachable / not JSON / channel absent → **fail open**: the + `optional` consumer contributes no rows and `data/visualeffects/lock.json` IDs + are preserved. +- vfxs.yml network error, or 404 **and** the `release_marker_path` (haks.json) + is **absent** for that tag (no published release) → **fail open**. +- vfxs.yml 404 **and** haks.json **present** → **HARD fail** ("release `` + has haks.json but no vfxs.yml — accessory rows would silently vanish"). +- vfxs.yml present but malformed / no `assets` array → **HARD fail**. + +Fail-open writes nothing and lets the optional consumer skip; hard-fail aborts +the build with the message above. + +**Deleted:** `sow-topdata/scripts/resolve-accessory-vfx.sh`, +`tests/resolve-accessory-vfx-contract.sh`, the `manifest_file:` key, and the +resolver pre-step at `build-topdata.sh:19`. The four bash contract cases +(offline override, unreachable fail-open, broken-release hard-fail, no-release +fail-open) are re-expressed as Go tests in `internal/topdata`. + +### B. Git-LFS materialization moves into Crucible + +So a bare clone + `crucible` packs real bytes, not pointer stubs (today only +`build-topdata.sh` pulls LFS, so direct Crucible would ship stubs in CI's +non-smudging checkout). + +Before packing assets into a hak, Crucible: detects git-LFS pointer stubs under +the asset tree; if any, runs `git lfs install --local` then `git lfs pull` in +the repo; re-checks and **hard-fails** if stubs remain ("refusing to build from +pointer stubs"). Uses the clone's own credentials (R1). Requires `git` + +`git-lfs` on `PATH` — provided by the Nix devshell, a standard host install +otherwise (a clone without git-lfs already yields stubs, so this is the existing +host expectation, now enforced loudly by Crucible instead of a wrapper). + +A skip control replaces `TOPDATA_SKIP_LFS` for the maintain-tree flow, which +regenerates the `data/` tree and **discards** the package: a Crucible flag/env +(e.g. `--skip-lfs` / `CRUCIBLE_SKIP_LFS`) so that one job can opt out. Default +is fail-closed (pull). + +### C. Wrappers thin out + +`scripts/build-topdata.sh` (and the shared pattern in every Crucible repo) +reduces to: validate (gate) → resolve crucible → `crucible `. No input +resolution, no LFS pull. `sow-module/scripts/package-module.sh` already matches +this shape (validate → resolve → build); it stays the reference. + +### D. CI parity guard (catches the next regression) + +A `sow-tools` integration test builds the topdata package via Crucible **alone** +in a bare checkout (no wrapper, clean `.cache`, no `NWN_ROOT`, anonymous CDN) +and asserts `visualeffects.2da` contains accessory rows. This is the test that +would have caught today's bug: it fails if any build-output-affecting step ever +drifts back out of Crucible into a wrapper, and it enforces R2/R3/R5 mechanically. + +The contract rule in `consumer-contract.md` is the written half; this test is +the enforced half. + +### E. NWN_ROOT cleanup (stale, contradicts the goal) + +`NWN_ROOT` is unused — no `os.Getenv`, no `--nwn-root` flag, no consumer. The +only traces are documentation/help-text describing a rule for a thing that does +not exist: the `## NWN_ROOT rule` section in `consumer-contract.md`, the help +text in `internal/dispatch/dispatch.go` ("if a builder ever needs an NWN +root… crucible never guesses from $HOME"), and the `README.md` pointer. That +old rule — *never guess, require an explicit env* — directly contradicts the +zero-config goal (R2): it would make a future builder demand a hand-set path. + +- **Remove** the `NWN_ROOT` references: the `consumer-contract.md` section, the + dispatch help-text lines, the README pointer. Leave the + `migration-from-nwn-tool.md` bullet as historical record. +- **Replace the forward rule.** Should a future builder ever need NWN game data, + it **auto-detects** it (the NWN home — containing `modules/`, `hak/`, `tlk/` — + and the Steam install path are reliably discoverable), with an optional + explicit override for non-standard layouts. Auto-detect first, never a required + hand-set env. This keeps "clone → run → Just Work" intact even for a builder + that one day reads game data. None do today. + +### F. Toolchain convergence — one Crucible, three modes + +Parity needs everyone to run the *same* Crucible. Today three mechanisms +diverge: non-Nix uses the `crucible.sh` bootstrap download; sow-assets-manifest's +flake pins `crucible v0.2.5` via `fetchurl`+hash; sow-topdata's flake ships no +crucible at all (falls back to the bootstrap, with a needless `CRUCIBLE_TOKEN`). +The pin is scattered (a hardcoded flake version, none in another flake, an image +sha in `prod.yml`, a HEAD-floating CI binary build). + +Converge on **one binary, one pin, three modes — all anonymous, all identical**: + +1. **Non-Nix (local + CI fallback).** `crucible.sh` / `crucible.ps1` bootstrap + downloads the release binary anonymously. `CRUCIBLE_TOKEN` is dropped from + consumer CI (R1). +2. **Nix (local + CI optimized).** Every consumer flake provides `crucible` in + its devshell from a **sow-tools flake input**, pinned by that repo's + `flake.lock`. Idiomatic, reproducible, no manual hashes; bump with + `nix flake update`. This replaces sow-assets-manifest's manual `fetchurl` + pin and **adds crucible to sow-topdata's devshell** (today it has none). +3. **CI** runs the *same* `crucible ` as local, inside the nix devshell + (binary-cache fast). No build container, no token, no CI-only pre-steps. + +**Pin source of truth = `flake.lock` per repo**, bumped deliberately in a PR. We +**reject floating "latest"**: it would let local drift from CI between releases +and break reproducibility. The non-Nix bootstrap, which resolves "latest" by +default, is the convenience path; reproducible/parity-critical builds pin (nix +devshell, or `CRUCIBLE_*` pin env for the bootstrap). + +### G. The crucible container is deployment-only + +`sow-tools` builds and publishes `registry.westgate.pw/deployment/crucible:` +on `v*` tags (`nix build .#image` → `skopeo`). It exists for **deployment** — +`prod.yml` pins it so tools travel with the runtime host (`nwn.enable`), and it +is a **disabled placeholder** until the NWN stack lands. No build/CI job consumes +it, and the host-mode runners have no container runtime to run it. It is **not a +build tool.** + +- Keep the image build (cheap, `v*`-gated) for its deployment purpose. +- **Fix `consumer-contract.md`:** remove the crossed-out "run CI inside the + pinned image" guidance and its OPERATOR NOTE; state plainly that builds use the + binary (bootstrap or nix devshell) and the image is deployment-only. This kills + the "why do we build a container we don't use?" confusion. + +### H. Autogen lock identity — model-anchored, immutable + +**General lock contract (hard rule, all datasets).** A dataset lock maps a stable +key to a row id. The build MUST: reuse the existing id for a key that still +exists; allocate the first free id (lexicographic 0→1→2…) for a new key; and free +an id only when its key is gone everywhere. An id, once assigned to a live key, +never changes. (Already implemented for the accessory consumer via +`historicalLockData` reuse + `nextAvailableAutogenID`; this section makes it the +written rule and fixes the autogen-specific anchor below.) + +**The autogen problem.** Autogen rows describe models that do **not** exist in +`sow-topdata/data/` — they live in the asset `vfxs.yml` manifest. Today the lock +key is the *config-derived presentation key*: +`KeyFormat = {dataset}:{group}{delimiter}{category_segment}{stem}`. Group token, +category, delimiter, case, and prefix-stripping all come from `nwn-tool.yaml`, so +a **config-only** change (`key_format`, `delimiter`, `case`, a group rename, +`category_from`, `strip_model_prefixes`) changes the key → the lock no longer +matches → a new id is allocated → **every row shifts**. That is a config edit +silently reshuffling ids for models that never changed. + +**Fix — anchor the lock identity on the dataset + model, not the presentation.** +The lock key for an autogen row becomes `{dataset}:{model_source_path}` — the +stable `dataset` namespace prefix (kept as the 2da/namespace linker developers use +instead of raw ids) plus the **model source path** from `vfxs.yml`, e.g. +`visualeffects:head_accessories/hat/hfx_bandana.mdl` (leading `vfxs/` stripped — +the path is the value already on `entry.Source`). Dropped from the key are only +the config-derived parts (group token, `category_segment`, `delimiter`, case, +prefix-stripping). The row's `key`/`label` *columns* still derive from config via +`KeyFormat`/`LabelFormat` — only the **lock identity** is decoupled and +model-anchored. Consequences: + +- A config-only change keeps every id (the path is unchanged). +- A model re-export (same path, new `sha256`) keeps its id — a content change is + not an identity change. (Audit trail comes free from `git diff` on the committed + `vfxs.yml`, which carries `path` + `sha256`; the lock stays a flat `string→int` + map — no schema change, no fork of the shared lock format. A future `id`+`sha` + object form is a clean later add if observability ever needs it, without + changing id semantics.) +- A renamed/moved/removed model is a genuinely different (or absent) identity. + +**One-time cutover remap.** On the first build under `{dataset}:{path}` keys, for +each current model compute its *old* config-key (with the current policy) and its +new `{dataset}:{path}` key; if the new key is absent but the old key carries an id +in the lock, adopt that id under the new key. So ids carry over with zero shift on +cutover; the now-unreferenced old keys fall stale and are pruned (below). + +**Stale pruning.** A lock key absent from the resolved entry set is dropped and +its id freed for first-free reuse — but **only on a successful, non-empty +resolution**. The fail-open paths (section A) contribute no entries and skip the +augmentor entirely, so an unreachable/missing manifest can never mass-prune the +lock. This realizes the general "free a fully-stale id" rule without risking the +lock on a transient source outage. + +**Testing.** Go tests in `internal/topdata`: config-only change (rename a group +token, change `key_format`/`delimiter`/`case`) → ids unchanged; model re-export +(same path, different sha) → id unchanged; model rename/remove → old id freed and +reusable; cutover remap → existing ids preserved across the key-scheme switch; +fail-open → no pruning, lock untouched. + +## Determinism vs. channel mutability (decision) + +The committed topdata commit pins everything **except** the asset channel +pointer. `current` resolves through `channels.json`, which is intentionally +mutable, so "reproducible" here means **same commit + same channel state → +identical bytes**, not "this commit is frozen forever." This is deliberate: the +accessory rows must track the asset HAKs shipped beside `sow_top.hak`, which is +exactly what the channel pointer expresses. A given asset release **tag** is +immutable; the pointer moving is a real input change, and `lock.json` preserves +accessory IDs across it (no reshuffle). Builds against an explicit tag +(`SOW_TOPDATA_ASSET_CHANNEL` set, or a `v*` ref) are fully pinned. + +This satisfies R1–R5: no secret, no config, deterministic for a fixed input +set, on any system. + +## Testing + +- **Go unit tests (`sow-tools/internal/topdata`)** for the `cdn_channel` + resolution: offline-override path; unreachable channels.json → fail open; + channel absent → fail open; vfxs.yml 404 + no haks.json → fail open; vfxs.yml + 404 + haks.json present → hard fail (asserts message); malformed vfxs.yml → + hard fail; happy path → correct filtered/sorted entries (mdl-only, 4-groups- + only, `vfxs/` stripped, group/subgroup/stem). Mirrors the deleted bash + contract cases. +- **Go config validation test** for the new `source` block. +- **CI parity guard** (section D) in the topdata build workflow. +- Existing accessory augmentor / lock-preserve tests unchanged. + +## Rollout + +1. sow-tools: add `AutogenSourceConfig`, the `cdn_channel` resolution branch, + LFS materialization, the model-anchored lock identity + cutover remap + stale + pruning (section H), the parity-guard test; update `consumer-contract.md` + (add the parity contract, remove the `NWN_ROOT` rule per section E, replace the + image guidance with deployment-only per section G). +2. Release Crucible (assets already anon-downloadable — R1). +3. Toolchain convergence (section F): expose `crucible` from a sow-tools flake + input in each consumer flake — add it to sow-topdata's devshell, switch + sow-assets-manifest off its manual `fetchurl` pin; drop `CRUCIBLE_TOKEN` from + consumer CI; CI builds inside the nix devshell. +4. sow-topdata: swap `manifest_file:` → `source:` in `nwn-tool.yaml`; delete the + resolver script + bash contract test; thin `build-topdata.sh` (validate → + resolve crucible → build); pin Crucible via `flake.lock`. +5. Verify: bare clone → `crucible topdata build-topdata` → 637 accessory rows, + no env, no key; and the same command inside `nix develop`. + +## Follow-ups (not this work) + +- Parts `2da` / `cachedmodels` reinstatement (separate spec) can reuse the same + `cdn_channel` source type. +- Roll the thinned-wrapper pattern review across `sow-module` / `sow-codebase`. diff --git a/internal/app/app.go b/internal/app/app.go index 0a606bb..324af93 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1620,16 +1620,20 @@ func runBuildTopData(ctx context) error { return err } - opts, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:]) + parsed, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:]) if err != nil { return err } + if parsed.skipLFS { + os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck + } + console := newTopdataConsole(ctx, p, "build-topdata") spin.configure(ctx.stderr, console.spinnerEnabled) spin.start("Build Topdata: starting") defer spin.stop() - result, err := topdata.BuildAndPackageWithOptions(p, opts, console.progress) + result, err := topdata.BuildAndPackageWithOptions(p, parsed.opts, console.progress) if err != nil { return err } @@ -1644,15 +1648,19 @@ func runBuildTopPackage(ctx context) error { return err } - opts, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:]) + parsed, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:]) if err != nil { return err } - if opts.BuildWiki { + if parsed.opts.BuildWiki { return fmt.Errorf("--wiki is not supported with build-top-package; use build-topdata when wiki generation is required") } + if parsed.skipLFS { + os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck + } + console := newTopdataConsole(ctx, p, "build-top-package") spin.configure(ctx.stderr, console.spinnerEnabled) spin.start("Build Top Package: starting") @@ -1666,21 +1674,30 @@ func runBuildTopPackage(ctx context) error { return nil } -func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndPackageOptions, error) { - opts := topdata.BuildAndPackageOptions{} +type buildTopDataArgs struct { + opts topdata.BuildAndPackageOptions + skipLFS bool +} + +func parseBuildTopDataArgs(commandName string, args []string) (buildTopDataArgs, error) { + var parsed buildTopDataArgs for _, arg := range args { switch arg { case "--force": - opts.Force = true + parsed.opts.Force = true case "--wiki": - opts.BuildWiki = true + parsed.opts.BuildWiki = true + case "--skip-lfs": + // ponytail: CRUCIBLE_SKIP_LFS env is the single skip mechanism; set it at + // the CLI entry point rather than carrying a dead field through BuildAndPackageOptions. + parsed.skipLFS = true case "-h", "--help": - return opts, fmt.Errorf("usage: %s [--force] [--wiki]", commandName) + return parsed, fmt.Errorf("usage: %s [--force] [--wiki] [--skip-lfs]", commandName) default: - return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) + return parsed, fmt.Errorf("unknown %s argument %q", commandName, arg) } } - return opts, nil + return parsed, nil } func runCompareTopData(ctx context) error { diff --git a/internal/dispatch/dispatch.go b/internal/dispatch/dispatch.go index 18fd98b..039a22c 100644 --- a/internal/dispatch/dispatch.go +++ b/internal/dispatch/dispatch.go @@ -249,8 +249,8 @@ func usage(w io.Writer) { fmt.Fprintf(w, " config [args] inspect/validate effective configuration\n") fmt.Fprintf(w, " changelog [args] generate the release changelog\n") fmt.Fprintf(w, "\nBuilders read all inputs from the project tree (nwn-tool.yaml + data/); no\n") - fmt.Fprintf(w, "NWN install is required. If a builder ever needs an NWN root, it must be\n") - fmt.Fprintf(w, "passed explicitly via NWN_ROOT; crucible never guesses from $HOME. See\n") + fmt.Fprintf(w, "NWN install is required. A builder that one day needs NWN game data\n") + fmt.Fprintf(w, "auto-detects the install; it never requires a hand-set path. See\n") fmt.Fprintf(w, "docs/consumer-contract.md.\n") } diff --git a/internal/project/project.go b/internal/project/project.go index 9aaccec..40ba9c7 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -366,6 +366,7 @@ type AutogenConsumerConfig struct { Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"` LocalOverrideRoot string `json:"local_override_root,omitempty" yaml:"local_override_root,omitempty"` ManifestFile string `json:"manifest_file,omitempty" yaml:"manifest_file,omitempty"` + Source AutogenSourceConfig `json:"source,omitempty" yaml:"source,omitempty"` } type AccessoryVisualeffectsConfig struct { @@ -416,6 +417,17 @@ type AutogenDeriveConfig struct { GroupFrom string `json:"group_from,omitempty" yaml:"group_from,omitempty"` } +type AutogenSourceConfig struct { + Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` + CDNBase string `json:"cdn_base,omitempty" yaml:"cdn_base,omitempty"` + CDNBaseEnv string `json:"cdn_base_env,omitempty" yaml:"cdn_base_env,omitempty"` + ChannelsPath string `json:"channels_path,omitempty" yaml:"channels_path,omitempty"` + ManifestPath string `json:"manifest_path,omitempty" yaml:"manifest_path,omitempty"` + ReleaseMarkerPath string `json:"release_marker_path,omitempty" yaml:"release_marker_path,omitempty"` + ChannelEnv string `json:"channel_env,omitempty" yaml:"channel_env,omitempty"` + OfflineOverrideEnv string `json:"offline_override_env,omitempty" yaml:"offline_override_env,omitempty"` +} + type AutogenManifestConfig struct { ReleaseTag string `json:"release_tag" yaml:"release_tag"` AssetName string `json:"asset_name" yaml:"asset_name"` @@ -641,6 +653,7 @@ func (p *Project) ValidateLayout() error { failures = append(failures, validateTopDataRowGeneration(effective.TopData.RowGeneration)...) failures = append(failures, validateTopDataRowExtensions(effective.TopData.RowExtensions)...) failures = append(failures, validateTopDataClassFeatInjections(effective.TopData.ClassFeatInjections)...) + failures = append(failures, validateAutogenConsumerSources(effective.Autogen.Consumers)...) failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...) failures = append(failures, validateRelativePath("topdata.compiled_tlk", effective.TopData.CompiledTLK)...) failures = append(failures, validateRelativePath("topdata.wiki.output_root", effective.TopData.Wiki.OutputRoot)...) @@ -1669,10 +1682,12 @@ func validateAutogenConfig(cfg AutogenConfig) []error { if strings.TrimSpace(consumer.Mode) == "accessory_visualeffects" && accessoryVisualeffectsConfigConfigured(consumer.AccessoryVisualeffects) { failures = append(failures, validateAccessoryVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", consumer.AccessoryVisualeffects)...) } - // A manifest_file consumer reads a pre-resolved local manifest, so the - // released-source fields (root/include/derive/manifest) are never read + // A manifest_file consumer reads a pre-resolved local manifest, and a + // cdn_channel consumer resolves entirely from Source.* at runtime, so + // neither reads the released-source fields (root/include/derive/manifest) // at build time — don't require them. See resolveAutogenConsumerManifest. - if strings.TrimSpace(consumer.ManifestFile) == "" { + if strings.TrimSpace(consumer.ManifestFile) == "" && + strings.TrimSpace(consumer.Source.Kind) != "cdn_channel" { if strings.TrimSpace(consumer.Root) == "" { failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix)) } @@ -1688,6 +1703,29 @@ func validateAutogenConfig(cfg AutogenConfig) []error { return failures } +func validateAutogenConsumerSources(consumers []AutogenConsumerConfig) []error { + var failures []error + for _, c := range consumers { + if strings.TrimSpace(c.Source.Kind) != "cdn_channel" { + continue + } + label := strings.TrimSpace(c.ID) + if label == "" { + label = "" + } + if strings.TrimSpace(c.Source.ChannelsPath) == "" { + failures = append(failures, fmt.Errorf("autogen consumer %s: source.channels_path is required for kind cdn_channel", label)) + } + manifestPath := strings.TrimSpace(c.Source.ManifestPath) + if manifestPath == "" { + failures = append(failures, fmt.Errorf("autogen consumer %s: source.manifest_path is required for kind cdn_channel", label)) + } else if !strings.Contains(manifestPath, "{tag}") { + failures = append(failures, fmt.Errorf("autogen consumer %s: source.manifest_path must contain {tag}", label)) + } + } + return failures +} + func validateGeneratedConfig(cfg GeneratedConfig) []error { var failures []error seen := map[string]struct{}{} diff --git a/internal/project/project_test.go b/internal/project/project_test.go index ca1b8f9..9c10e9c 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -1425,6 +1425,51 @@ func TestValidateLayoutRejectsInvalidAccessoryVisualeffectsNamingConfig(t *testi } } +func TestValidateLayoutAcceptsCDNChannelConsumerWithoutReleasedSourceFields(t *testing.T) { + // cdn_channel consumers resolve entirely from Source.* at runtime; they must + // not be required to supply root/include/derive/manifest fields. + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Autogen: AutogenConfig{ + Consumers: []AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Optional: true, + AccessoryVisualeffects: AccessoryVisualeffectsConfig{ + Groups: map[string]AccessoryVisualeffectGroupConfig{ + "head_accessories": {Prefix: "head_acc_"}, + "chest_accessories": {Prefix: "chest_acc_"}, + "head_decorations": {Prefix: "head_dec_"}, + "head_features": {Prefix: "head_feat_"}, + }, + }, + Source: AutogenSourceConfig{ + Kind: "cdn_channel", + ChannelsPath: "releases/haks/channels.json", + ManifestPath: "releases/haks/{tag}/vfxs.yml", + }, + // Intentionally omitted: Root, Include, Derive, Manifest + }, + }, + }, + }, + } + + if err := proj.ValidateLayout(); err != nil { + t.Fatalf("cdn_channel consumer should pass ValidateLayout without released-source fields, got: %v", err) + } +} + func TestLoadRejectsLegacyAccessoryVisualeffectsNamingFields(t *testing.T) { root := t.TempDir() writeProjectFile(t, filepath.Join(root, ConfigFile), ` @@ -1796,6 +1841,31 @@ func TestCloneWithHAKNamesRejectsUnknownHAKs(t *testing.T) { } } +func TestValidateAutogenConsumerSourcesCDNChannel(t *testing.T) { + cases := []struct { + name string + source AutogenSourceConfig + wantFail bool + }{ + {"valid", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json", ManifestPath: "releases/haks/{tag}/vfxs.yml"}, false}, + {"missing channels_path", AutogenSourceConfig{Kind: "cdn_channel", ManifestPath: "releases/haks/{tag}/vfxs.yml"}, true}, + {"missing manifest_path", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json"}, true}, + {"manifest_path lacks {tag}", AutogenSourceConfig{Kind: "cdn_channel", ChannelsPath: "releases/haks/channels.json", ManifestPath: "releases/haks/vfxs.yml"}, true}, + {"empty kind is ignored", AutogenSourceConfig{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + failures := validateAutogenConsumerSources([]AutogenConsumerConfig{{ID: "x", Source: tc.source}}) + if tc.wantFail && len(failures) == 0 { + t.Fatalf("expected validation failure, got none") + } + if !tc.wantFail && len(failures) != 0 { + t.Fatalf("expected no failure, got %v", failures) + } + }) + } +} + func loadAndValidate(root string) error { proj, err := Load(root) if err != nil { diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go index 37dc846..9168cbd 100644 --- a/internal/topdata/autogen.go +++ b/internal/topdata/autogen.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "os" "path/filepath" "slices" @@ -13,10 +14,13 @@ import ( "time" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gopkg.in/yaml.v3" ) const autogenManifestCacheMaxAge = time.Hour +const defaultBunnyCDNBase = "https://depot.westgate.pw" + // errAutogenManifestUnavailable marks a released autogen manifest that could not // be located or fetched (network failure, missing release/asset, empty payload, // or an undeterminable sow-assets repo). It is deliberately distinct from an @@ -178,17 +182,22 @@ func preserveAutogenConsumerLockEntries(collected []nativeCollectedDataset, cons func autogenConsumerManagedLockKeyMatcher(consumer project.AutogenConsumerConfig) func(string) bool { switch consumer.Mode { case "accessory_visualeffects": - policy := resolveAccessoryVisualeffectsPolicy(consumer, nil) - delimiter := policy.Delimiter - if delimiter == "" { - delimiter = "/" + dataset := strings.TrimSpace(consumer.Dataset) + if dataset == "" { + dataset = "visualeffects" } - prefix := "visualeffects:" + prefix := dataset + ":" + // Build the group set from the resolved policy so consumers that omit + // AccessoryVisualeffects.Groups still match the four default group names. + // NOTE: this matcher assumes the default folder_name+preserve group + // representation. Keys produced under case:lower or + // group_token_source:prefix would not match and would escape pruning; + // no production config uses those combinations. + policy := resolveAccessoryVisualeffectsPolicy(consumer, nil) groups := make(map[string]struct{}, len(policy.Groups)) - for group, groupPolicy := range policy.Groups { - token := applyAccessoryVisualeffectCase(accessoryVisualeffectGroupToken(group, groupPolicy, policy), policy) - if strings.TrimSpace(token) != "" { - groups[token] = struct{}{} + for group := range policy.Groups { + if group = strings.TrimSpace(group); group != "" { + groups[group] = struct{}{} } } if len(groups) == 0 { @@ -199,7 +208,7 @@ func autogenConsumerManagedLockKeyMatcher(consumer project.AutogenConsumerConfig return false } rest := key[len(prefix):] - idx := strings.Index(rest, delimiter) + idx := strings.Index(rest, "/") if idx <= 0 { return false } @@ -225,6 +234,9 @@ func autogenConsumerTargetsDataset(dataset nativeCollectedDataset, consumer proj } func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) { + if strings.TrimSpace(consumer.Source.Kind) == "cdn_channel" { + return resolveCDNChannelManifest(p, consumer, consumer.Source, progress) + } if manifestFile := strings.TrimSpace(consumer.ManifestFile); manifestFile != "" { return readAutogenConsumerManifestFile(p, consumer, manifestFile, progress) } @@ -276,6 +288,183 @@ func readAutogenConsumerManifestFile(p *project.Project, consumer project.Autoge return &manifest, nil } +// resolveCDNChannelManifest resolves the accessory-VFX model list anonymously +// from the asset CDN: channel pointer -> tag -> per-tag vfxs.yml. It ports the +// fail policy of the deleted resolve-accessory-vfx.sh: unreachable inputs fail +// OPEN (errAutogenManifestUnavailable -> optional consumer preserves lock IDs); +// a published-but-broken release (vfxs.yml 404 while haks.json present) or a +// malformed vfxs.yml HARD fail. +func resolveCDNChannelManifest(p *project.Project, consumer project.AutogenConsumerConfig, src project.AutogenSourceConfig, progress func(string)) (*autogenManifest, error) { + if progress == nil { + progress = func(string) {} + } + + // 1. Offline / air-gapped override: a vfxs.yml path or a manifest-repo + // checkout root containing assets/vfxs.yml. Skips the network entirely. + if envName := strings.TrimSpace(src.OfflineOverrideEnv); envName != "" { + if override := strings.TrimSpace(os.Getenv(envName)); override != "" { + vfxsPath := override + if info, err := os.Stat(override); err == nil && info.IsDir() { + vfxsPath = filepath.Join(override, "assets", "vfxs.yml") + } + raw, err := os.ReadFile(vfxsPath) + if err != nil { + return nil, fmt.Errorf("offline vfxs override %s: %w", vfxsPath, err) + } + entries, err := filterCDNChannelEntries(raw, consumer) + if err != nil { + return nil, err + } + progress(fmt.Sprintf("Using offline vfxs override for %s from %s...", consumer.ID, vfxsPath)) + return &autogenManifest{ID: consumer.Producer, Ref: "local", Entries: entries}, nil + } + } + + // 2. Channel: explicit env, else derived from the git tag. + channel := strings.TrimSpace(os.Getenv(strings.TrimSpace(src.ChannelEnv))) + if channel == "" { + channel = deriveAssetChannel(p.Root) + } + + // 3. CDN base: baked default, then env override, then config literal (most specific wins). + cdnBase := defaultBunnyCDNBase + if envName := strings.TrimSpace(src.CDNBaseEnv); envName != "" { + if v := strings.TrimSpace(os.Getenv(envName)); v != "" { + cdnBase = v + } + } + if v := strings.TrimSpace(src.CDNBase); v != "" { + cdnBase = v + } + cdnBase = strings.TrimRight(cdnBase, "/") + + join := func(path, tag string) string { + return cdnBase + "/" + strings.TrimLeft(strings.ReplaceAll(path, "{tag}", tag), "/") + } + + // 4. channels.json -> tag. Unreachable / non-JSON / channel-absent = fail open. + channelsURL := join(src.ChannelsPath, "") + status, body, err := httpGetStatus(channelsURL) + if err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json unreachable %s: %w", channelsURL, err)) + } + if status != http.StatusOK { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json HTTP %d %s", status, channelsURL)) + } + var channels map[string]string + if err := json.Unmarshal(body, &channels); err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("channels.json not JSON %s: %w", channelsURL, err)) + } + tag := strings.TrimSpace(channels[channel]) + if tag == "" { + return nil, unavailableAutogenManifest(fmt.Errorf("channel %q absent in channels.json (%s)", channel, channelsURL)) + } + progress(fmt.Sprintf("Accessory VFX: channel %s -> tag %s", channel, tag)) + + // 5. vfxs.yml for the tag. + manifestURL := join(src.ManifestPath, tag) + vstatus, vbody, err := httpGetStatus(manifestURL) + if err != nil { + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml unreachable %s: %w", manifestURL, err)) + } + switch vstatus { + case http.StatusOK: + // fall through to parse + case http.StatusNotFound: + // 404 + release marker present = published-but-broken release: HARD fail + // (this is the v0.1.4 silent-drop bug). 404 + marker absent = no published + // release for this tag: fail open. + if marker := strings.TrimSpace(src.ReleaseMarkerPath); marker != "" { + markerURL := join(marker, tag) + if mstatus, _, merr := httpGetStatus(markerURL); merr == nil && mstatus == http.StatusOK { + return nil, fmt.Errorf("release %s has %s but no vfxs.yml (%s) — accessory rows would silently vanish; backfill/republish vfxs.yml for %s", tag, marker, manifestURL, tag) + } + } + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml 404 %s (no published release for %s)", manifestURL, tag)) + default: + return nil, unavailableAutogenManifest(fmt.Errorf("vfxs.yml HTTP %d %s", vstatus, manifestURL)) + } + + entries, err := filterCDNChannelEntries(vbody, consumer) + if err != nil { + return nil, err // malformed vfxs.yml = HARD fail + } + progress(fmt.Sprintf("Accessory VFX: resolved %d model entries from %s", len(entries), manifestURL)) + return &autogenManifest{ID: consumer.Producer, Ref: tag, Entries: entries}, nil +} + +// filterCDNChannelEntries parses vfxs.yml and keeps restype==mdl assets under +// vfxs// for the consumer's 4 accessory groups, stripping the leading +// vfxs/ from each source path. A present-but-malformed manifest (unparseable, or +// no assets array) is an error; an empty assets array yields zero entries. +func filterCDNChannelEntries(raw []byte, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) { + var manifest struct { + Assets *[]struct { + Path string `yaml:"path"` + Restype string `yaml:"restype"` + } `yaml:"assets"` + } + if err := yaml.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("malformed vfxs.yml (cannot parse): %w", err) + } + if manifest.Assets == nil { + return nil, fmt.Errorf("malformed vfxs.yml (no assets array)") + } + groups := make(map[string]struct{}, len(consumer.AccessoryVisualeffects.Groups)) + for g := range consumer.AccessoryVisualeffects.Groups { + if g = strings.TrimSpace(g); g != "" { + groups[g] = struct{}{} + } + } + var entries []autogenManifestEntry + for _, a := range *manifest.Assets { + if a.Restype != "mdl" { + continue + } + path := filepath.ToSlash(strings.TrimSpace(a.Path)) + if !strings.HasPrefix(path, "vfxs/") { + continue + } + rel := strings.TrimPrefix(path, "vfxs/") + parts := strings.Split(rel, "/") + if len(parts) < 2 { + continue + } + if _, ok := groups[parts[0]]; !ok { + continue + } + entry := autogenManifestEntry{ + Source: rel, + Group: parts[0], + ModelStem: strings.TrimSuffix(parts[len(parts)-1], ".mdl"), + } + if len(parts) > 2 { + entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") + } + entries = append(entries, entry) + } + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + return strings.Compare(a.Source, b.Source) + }) + return entries, nil +} + +// deriveAssetChannel maps the current git tag (or GITHUB_REF_NAME) to a channel: +// a prerelease tag (v*-*) -> testing; a stable tag (v*) -> current; anything +// else (branch/PR/dev) -> current. Mirrors resolve-accessory-vfx.sh. +func deriveAssetChannel(root string) string { + ref := strings.TrimSpace(os.Getenv("GITHUB_REF_NAME")) + if ref == "" { + if out, err := gitOutput(root, "describe", "--tags", "--exact-match"); err == nil { + ref = strings.TrimSpace(out) + } + } + if strings.HasPrefix(ref, "v") && strings.Contains(ref, "-") { + return "testing" + } + return "current" +} + func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) { root := strings.TrimSpace(consumer.LocalOverrideRoot) if root != "" { @@ -870,30 +1059,61 @@ func augmentWithAutogeneratedAccessoryVisualeffects(collected []nativeCollectedD return rowID } + liveLockKeys := make(map[string]struct{}, len(entries)) for _, entry := range entries { key, label, modelStem, groupPolicy, ok := accessoryVisualeffectIdentity(dataset.Dataset.Name, entry, policy) if !ok { continue } + lockKey := accessoryVisualeffectLockKey(dataset.Dataset.Name, entry) + liveLockKeys[lockKey] = struct{}{} + if existing, exists := rowByKey[key]; exists { applyDiscoveredAccessoryVisualeffectDefaults(existing, dataset.Columns, modelStem, label, policy, groupPolicy) + if _, pinned := lockData[lockKey]; !pinned { + if rowID, ok := existing["id"].(int); ok { + lockData[lockKey] = rowID + } + } continue } - rowID, ok := lockData[key] + rowID, ok := lockData[lockKey] if !ok { - if preservedRowID, preserved := historicalLockData[key]; preserved { - rowID = preservedRowID - } else { + switch { + case hasKey(historicalLockData, lockKey): + rowID = historicalLockData[lockKey] + case hasKey(historicalLockData, key): + // One-time cutover: adopt the ID from the old config-derived key + // so IDs carry over with zero shift on the key-scheme switch. The + // stale old key is pruned below. + rowID = historicalLockData[key] + default: rowID = allocateNextID() } - lockData[key] = rowID + lockData[lockKey] = rowID } newRow := createDefaultAccessoryVisualeffectRow(dataset.Columns, rowID, key, label, modelStem, policy, groupPolicy) rows = append(rows, newRow) rowByKey[key] = newRow } + // Stale pruning: drop managed lock keys (this consumer's accessory keys, + // incl. now-superseded old config-scheme keys) that no live entry resolves + // to, freeing their IDs for first-free reuse. Safe here because the + // augmentor only runs on a successful, non-empty resolution; the fail-open + // paths skip it entirely and never reach this code. + if managed := autogenConsumerManagedLockKeyMatcher(consumer); managed != nil { + for lk := range lockData { + if _, live := liveLockKeys[lk]; live { + continue + } + if managed(lk) { + delete(lockData, lk) + } + } + } + slices.SortFunc(rows, func(a, b map[string]any) int { return a["id"].(int) - b["id"].(int) }) @@ -912,6 +1132,20 @@ func nextAvailableAutogenID(used map[int]struct{}) int { } } +// accessoryVisualeffectLockKey is the model-anchored autogen lock identity: +// the dataset namespace prefix plus the model source path from vfxs.yml (leading +// vfxs/ already stripped on entry.Source). Config-derived presentation parts +// (group token, category, delimiter, case, prefix-stripping) are deliberately +// excluded so a config-only edit never reshuffles row IDs. +func accessoryVisualeffectLockKey(dataset string, entry autogenManifestEntry) string { + return dataset + ":" + strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(entry.Source)), "vfxs/") +} + +func hasKey(m map[string]int, key string) bool { + _, ok := m[key] + return ok +} + type accessoryVisualeffectsPolicy struct { Groups map[string]accessoryVisualeffectsGroupPolicy GroupTokenSource string diff --git a/internal/topdata/autogen_cdn_channel_test.go b/internal/topdata/autogen_cdn_channel_test.go new file mode 100644 index 0000000..efa042a --- /dev/null +++ b/internal/topdata/autogen_cdn_channel_test.go @@ -0,0 +1,199 @@ +package topdata + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func cdnConsumer() project.AutogenConsumerConfig { + return project.AutogenConsumerConfig{ + ID: "accessory_visualeffects", Producer: "accessory_visualeffects", + Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true, + AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{ + Groups: map[string]project.AccessoryVisualeffectGroupConfig{ + "head_accessories": {}, "chest_accessories": {}, + "head_decorations": {}, "head_features": {}, + }, + }, + Source: project.AutogenSourceConfig{ + Kind: "cdn_channel", + ChannelsPath: "releases/haks/channels.json", + ManifestPath: "releases/haks/{tag}/vfxs.yml", + ReleaseMarkerPath: "releases/haks/{tag}/haks.json", + }, + } +} + +// cdnServer serves channels.json, vfxs.yml, and haks.json from in-memory maps. +// A key absent from the map returns 404. +func cdnServer(t *testing.T, files map[string]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(w, body) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestResolveCDNChannelHappyPath(t *testing.T) { + root := testProjectRoot(t) + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1.2.3"}`, + "/releases/haks/v1.2.3/vfxs.yml": `assets: + - path: vfxs/head_accessories/hat/hfx_bandana.mdl + restype: mdl + sha256: aaa + - path: vfxs/chest_accessories/cape/cfx_cloak.mdl + restype: mdl + sha256: bbb + - path: vfxs/head_accessories/hat/hfx_bandana.tga + restype: tga + sha256: ccc + - path: vfxs/weapons/sword.mdl + restype: mdl + sha256: ddd +`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + p := testProject(root) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + + m, err := resolveCDNChannelManifest(p, c, c.Source, nil) + if err != nil { + t.Fatalf("resolve failed: %v", err) + } + if len(m.Entries) != 2 { + t.Fatalf("want 2 mdl entries under target groups, got %d: %#v", len(m.Entries), m.Entries) + } + // Sorted by source: chest_accessories/... before head_accessories/... + if m.Entries[0].Source != "chest_accessories/cape/cfx_cloak.mdl" { + t.Fatalf("unexpected first entry: %#v", m.Entries[0]) + } + if m.Entries[1].Source != "head_accessories/hat/hfx_bandana.mdl" || + m.Entries[1].Group != "head_accessories" || + m.Entries[1].Subgroup != "hat" || + m.Entries[1].ModelStem != "hfx_bandana" { + t.Fatalf("unexpected derived entry: %#v", m.Entries[1]) + } + if m.Ref != "v1.2.3" { + t.Fatalf("want ref v1.2.3, got %q", m.Ref) + } +} + +func TestResolveCDNChannelFailOpen(t *testing.T) { + root := testProjectRoot(t) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + t.Run("channels.json unreachable", func(t *testing.T) { + t.Setenv("BUNNY_CDN_BASE", "http://127.0.0.1:0") // unroutable + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) + + t.Run("channel absent", func(t *testing.T) { + srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"testing":"v9"}`}) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) + + t.Run("vfxs 404 no release marker", func(t *testing.T) { + srv := cdnServer(t, map[string]string{"/releases/haks/channels.json": `{"current":"v1"}`}) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || !errorIsUnavailable(err) { + t.Fatalf("want fail-open (unavailable), got %v", err) + } + }) +} + +func TestResolveCDNChannelHardFail(t *testing.T) { + root := testProjectRoot(t) + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + + t.Run("broken release: vfxs 404 + haks.json present", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/haks.json": `{}`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail, got %v", err) + } + }) + + t.Run("malformed vfxs.yml: no assets array", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/vfxs.yml": "generated_at: 2026-01-01\n", + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail, got %v", err) + } + }) + + t.Run("malformed vfxs.yml: syntactically broken YAML", func(t *testing.T) { + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1"}`, + "/releases/haks/v1/vfxs.yml": "assets:\n - path: [unterminated", + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + _, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err == nil || errorIsUnavailable(err) { + t.Fatalf("want HARD fail on broken YAML, got %v", err) + } + }) +} + +func TestResolveCDNChannelOfflineOverride(t *testing.T) { + root := testProjectRoot(t) + vfxs := filepath.Join(root, "vfxs.yml") + writeFile(t, vfxs, `assets: + - path: vfxs/head_features/scar/hfx_scar.mdl + restype: mdl + sha256: zzz +`) + t.Setenv("SOW_VFXS_MANIFEST", vfxs) + c := cdnConsumer() + c.Source.OfflineOverrideEnv = "SOW_VFXS_MANIFEST" + + m, err := resolveCDNChannelManifest(testProject(root), c, c.Source, nil) + if err != nil { + t.Fatalf("offline override failed: %v", err) + } + if len(m.Entries) != 1 || m.Entries[0].Source != "head_features/scar/hfx_scar.mdl" { + t.Fatalf("unexpected offline entries: %#v", m.Entries) + } +} + +func errorIsUnavailable(err error) bool { + return errors.Is(err, errAutogenManifestUnavailable) +} diff --git a/internal/topdata/autogen_lock_identity_test.go b/internal/topdata/autogen_lock_identity_test.go new file mode 100644 index 0000000..3feb059 --- /dev/null +++ b/internal/topdata/autogen_lock_identity_test.go @@ -0,0 +1,114 @@ +package topdata + +import ( + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func accConsumer() project.AutogenConsumerConfig { + return project.AutogenConsumerConfig{ + ID: "accessory_visualeffects", Producer: "accessory_visualeffects", + Dataset: "visualeffects", Mode: "accessory_visualeffects", Optional: true, + AccessoryVisualeffects: project.AccessoryVisualeffectsConfig{ + Groups: map[string]project.AccessoryVisualeffectGroupConfig{ + "head_accessories": {ModelColumns: []string{"Imp_HeadCon_Node"}}, + }, + GroupTokenSource: "folder_name", CategoryFrom: "immediate_parent", + Delimiter: "/", Case: "preserve", + StripModelPrefixes: []string{"hfx_"}, + KeyFormat: "{dataset}:{group}{delimiter}{category_segment}{stem}", + LabelFormat: "{group}{delimiter}{category_segment}{stem}", + ModelColumn: "Imp_HeadCon_Node", + }, + } +} + +func vfxEntry() autogenManifestEntry { + return autogenManifestEntry{ + Source: "head_accessories/hat/hfx_bandana.mdl", Group: "head_accessories", + Subgroup: "hat", ModelStem: "hfx_bandana", + } +} + +func vfxDataset(lock map[string]int) nativeCollectedDataset { + return nativeCollectedDataset{ + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Imp_HeadCon_Node"}, + Rows: nil, + LockData: lock, + } +} + +func TestAccessoryLockKeyIsModelAnchored(t *testing.T) { + got := accessoryVisualeffectLockKey("visualeffects", vfxEntry()) + want := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + if got != want { + t.Fatalf("lock key = %q, want %q", got, want) + } +} + +// A config-only change (here: a different key_format) must NOT move the ID, +// because identity is the model path, not the presentation key. +func TestConfigOnlyChangeKeepsLockID(t *testing.T) { + lockKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{lockKey: 555})} + + c := accConsumer() + c.AccessoryVisualeffects.KeyFormat = "{dataset}:{stem}" // config-only change + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, c, nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if got[0].LockData[lockKey] != 555 { + t.Fatalf("expected ID preserved at 555, got %#v", got[0].LockData) + } +} + +// A model re-export (same path) keeps its ID — content change != identity change. +func TestModelReExportKeepsLockID(t *testing.T) { + lockKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{lockKey: 777})} + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if got[0].LockData[lockKey] != 777 { + t.Fatalf("expected ID 777 preserved, got %#v", got[0].LockData) + } +} + +// A removed model frees its old ID; the surviving model keeps its lock entry. +func TestModelRemovalPrunesStaleLockKey(t *testing.T) { + keep := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + gone := "visualeffects:head_accessories/hat/hfx_removed.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{keep: 1, gone: 2})} + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if _, present := got[0].LockData[gone]; present { + t.Fatalf("expected removed model's lock key pruned, got %#v", got[0].LockData) + } + if got[0].LockData[keep] != 1 { + t.Fatalf("expected surviving model ID 1 kept, got %#v", got[0].LockData) + } +} + +// Cutover: a lock that only has the OLD config-derived key adopts that ID under +// the new model-path key, with zero shift, and prunes the old key. +func TestCutoverRemapPreservesID(t *testing.T) { + oldKey := "visualeffects:head_accessories/hat/bandana" // {dataset}:{group}/{category}/{stem-without-hfx_} + newKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + collected := []nativeCollectedDataset{vfxDataset(map[string]int{oldKey: 4242})} + got, err := augmentWithAutogeneratedAccessoryVisualeffects(collected, []autogenManifestEntry{vfxEntry()}, accConsumer(), nil) + if err != nil { + t.Fatalf("augment failed: %v", err) + } + if got[0].LockData[newKey] != 4242 { + t.Fatalf("expected cutover to carry ID 4242 to new key, got %#v", got[0].LockData) + } + if _, present := got[0].LockData[oldKey]; present { + t.Fatalf("expected old config key pruned after cutover, got %#v", got[0].LockData) + } +} diff --git a/internal/topdata/autogen_manifest_file_test.go b/internal/topdata/autogen_manifest_file_test.go index b581ec3..c99db46 100644 --- a/internal/topdata/autogen_manifest_file_test.go +++ b/internal/topdata/autogen_manifest_file_test.go @@ -57,7 +57,7 @@ func TestApplyAutogenConsumersReadsManifestFile(t *testing.T) { if row["key"] != "visualeffects:head_accessories/hat/bandana" || row["Imp_HeadCon_Node"] != "hfx_bandana" { t.Fatalf("unexpected generated row: %#v", row) } - if _, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok { + if _, ok := got[0].LockData["visualeffects:head_accessories/hat/hfx_bandana.mdl"]; !ok { t.Fatalf("expected lock id for generated accessory, got %#v", got[0].LockData) } // Authored key untouched. diff --git a/internal/topdata/lfs_materialize_test.go b/internal/topdata/lfs_materialize_test.go new file mode 100644 index 0000000..089465c --- /dev/null +++ b/internal/topdata/lfs_materialize_test.go @@ -0,0 +1,29 @@ +package topdata + +import ( + "os" + "path/filepath" + "testing" +) + +func TestHasLFSPointerStubsDetectsStub(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "a"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + stub := "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 12345\n" + writeFile(t, filepath.Join(root, "a", "model.mdl"), stub) + writeFile(t, filepath.Join(root, "a", "real.txt"), "not a pointer, real bytes here") + + if !hasLFSPointerStubs(root) { + t.Fatal("expected stub detection to be true") + } +} + +func TestHasLFSPointerStubsNoStub(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "real.mdl"), "binary-ish content without a pointer header") + if hasLFSPointerStubs(root) { + t.Fatal("expected no stub detected") + } +} diff --git a/internal/topdata/parity_guard_test.go b/internal/topdata/parity_guard_test.go new file mode 100644 index 0000000..0408f85 --- /dev/null +++ b/internal/topdata/parity_guard_test.go @@ -0,0 +1,56 @@ +package topdata + +import ( + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +// PARITY GUARD: a bare project (no manifest_file, no NWN_ROOT, no token) with a +// cdn_channel consumer must produce accessory rows. If accessory resolution ever +// drifts back out of Crucible into a wrapper, this fails. +func TestParityGuardCDNChannelProducesAccessoryRows(t *testing.T) { + root := testProjectRoot(t) + srv := cdnServer(t, map[string]string{ + "/releases/haks/channels.json": `{"current":"v1.0.0"}`, + "/releases/haks/v1.0.0/vfxs.yml": `assets: + - path: vfxs/head_accessories/hat/hfx_bandana.mdl + restype: mdl + sha256: aaa + - path: vfxs/chest_accessories/cape/cfx_cloak.mdl + restype: mdl + sha256: bbb +`, + }) + t.Setenv("BUNNY_CDN_BASE", srv.URL) + t.Setenv("SOW_TOPDATA_ASSET_CHANNEL", "current") + // Prove R3: no NWN_ROOT in the environment. + t.Setenv("NWN_ROOT", "") + + c := cdnConsumer() + c.Source.CDNBaseEnv = "BUNNY_CDN_BASE" + c.Source.ChannelEnv = "SOW_TOPDATA_ASSET_CHANNEL" + c.AccessoryVisualeffects.ModelColumn = "Imp_HeadCon_Node" + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{c} + + collected := []nativeCollectedDataset{{ + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Imp_HeadCon_Node", "Imp_Root_S_Node"}, + Rows: nil, + LockData: map[string]int{}, + }} + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got[0].Rows) != 2 { + t.Fatalf("PARITY GUARD: expected 2 accessory rows, got %d (%#v)", len(got[0].Rows), got[0].Rows) + } + wantKey := "visualeffects:head_accessories/hat/hfx_bandana.mdl" + if _, ok := got[0].LockData[wantKey]; !ok { + t.Fatalf("PARITY GUARD: expected lock id for %s, got %#v", wantKey, got[0].LockData) + } +} diff --git a/internal/topdata/parts_manifest.go b/internal/topdata/parts_manifest.go index 725a95a..a5e11e0 100644 --- a/internal/topdata/parts_manifest.go +++ b/internal/topdata/parts_manifest.go @@ -274,6 +274,19 @@ func writePartsManifestCache(path string, manifest *partsManifest) error { return nil } +// httpGetStatus performs an anonymous GET and returns the status code plus the +// body (capped). No auth header: cdn_channel inputs (channels.json, vfxs.yml, +// haks.json) are public CDN text — R1. +func httpGetStatus(target string) (int, []byte, error) { + resp, err := partsManifestHTTPClient.Get(target) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + return resp.StatusCode, body, nil +} + func inventoryFromPartsManifest(manifest *partsManifest) map[string]map[int]struct{} { result := make(map[string]map[int]struct{}, len(supportedPartCategories)) for _, category := range supportedPartCategories { diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go index 76d9a85..bd9e5b0 100644 --- a/internal/topdata/top_package.go +++ b/internal/topdata/top_package.go @@ -1,6 +1,7 @@ package topdata import ( + "bytes" "errors" "fmt" "io" @@ -75,6 +76,8 @@ func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult, type BuildAndPackageOptions struct { Force bool BuildWiki bool + // ponytail: no SkipLFS field — CRUCIBLE_SKIP_LFS env is the single skip mechanism; + // the CLI flag sets it directly before calling into this package. } func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { @@ -123,6 +126,9 @@ func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress if progress == nil { progress = func(string) {} } + if err := materializeAssetLFS(p, progress); err != nil { + return PackageResult{}, err + } outputHAK := p.TopDataPackageHAKPath() outputTLK := p.TopDataPackageTLKPath() @@ -711,3 +717,80 @@ func countCompiledFiles(dir, extension string) (int, time.Time, error) { } return count, oldest, nil } + +// materializeAssetLFS ensures the topdata asset tree holds real bytes, not +// Git-LFS pointer stubs, before the hak is packed. A bare CI checkout does not +// smudge LFS, so a direct Crucible build would otherwise ship ~130-byte stubs. +// It uses the clone's own credentials (R1) and hard-fails if stubs survive the +// pull. Set CRUCIBLE_SKIP_LFS=1 to opt out (maintain-tree regenerates the data/ +// tree and discards the package, so it must not hard-depend on LFS). +func materializeAssetLFS(p *project.Project, progress func(string)) error { + if progress == nil { + progress = func(string) {} + } + assetsDir := filepath.Join(p.TopDataSourceDir(), "assets") + info, err := os.Stat(assetsDir) + if err != nil || !info.IsDir() { + return nil // no asset tree to materialize + } + if !hasLFSPointerStubs(assetsDir) { + return nil // already real bytes (or repo does not use LFS) + } + if isTruthyEnv(os.Getenv("CRUCIBLE_SKIP_LFS")) { + progress("CRUCIBLE_SKIP_LFS set: leaving Git-LFS pointer stubs unmaterialized (package will be discarded)") + return nil + } + if _, err := gitOutput(p.Root, "lfs", "version"); err != nil { + return fmt.Errorf("asset tree has Git-LFS pointer stubs but git-lfs is unavailable; install git-lfs or enter the nix devshell: %w", err) + } + progress("Materializing Git-LFS assets (git lfs pull)...") + // A fresh clone may lack the lfs smudge/clean filters; install them locally + // first (scoped to this repo's .git/config, no host mutation). Idempotent. + if _, err := gitOutput(p.Root, "lfs", "install", "--local"); err != nil { + return fmt.Errorf("git lfs install --local failed: %w", err) + } + if _, err := gitOutput(p.Root, "lfs", "pull"); err != nil { + return fmt.Errorf("git lfs pull failed: %w", err) + } + if hasLFSPointerStubs(assetsDir) { + return fmt.Errorf("refusing to build from pointer stubs: Git-LFS pointer stubs remain under %s after git lfs pull", assetsDir) + } + return nil +} + +// hasLFSPointerStubs reports whether any small text file under root is a Git-LFS +// pointer. Real assets are larger than any pointer stub, so the size gate keeps +// this cheap on a tree of binaries. +func hasLFSPointerStubs(root string) bool { + found := false + _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || found { + return nil + } + info, ierr := d.Info() + if ierr != nil || info.Size() == 0 || info.Size() > 1024 { + return nil + } + f, oerr := os.Open(path) + if oerr != nil { + return nil + } + defer f.Close() + buf := make([]byte, 256) + n, _ := f.Read(buf) + if bytes.Contains(buf[:n], []byte("git-lfs.github.com/spec/v1")) { + found = true + } + return nil + }) + return found +} + +func isTruthyEnv(v string) bool { + switch strings.TrimSpace(strings.ToLower(v)) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 4529cde..76fc42c 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -7706,13 +7706,13 @@ func TestApplyAutogenConsumersAugmentsAccessoryVisualeffectsFromLocalOverride(t t.Fatalf("unexpected head feature defaults: %#v", hair) } - if _, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok { + if _, ok := got[0].LockData["visualeffects:head_accessories/hat/hfx_bandana.mdl"]; !ok { t.Fatalf("expected head accessory lock id, got %#v", got[0].LockData) } - if _, ok := got[0].LockData["visualeffects:head_decorations/laurel/laurel"]; !ok { + if _, ok := got[0].LockData["visualeffects:head_decorations/laurel/hfx_laurel.mdl"]; !ok { t.Fatalf("expected head decoration lock id, got %#v", got[0].LockData) } - if _, ok := got[0].LockData["visualeffects:head_features/hair/hair_bangs"]; !ok { + if _, ok := got[0].LockData["visualeffects:head_features/hair/hfx_hair_bangs.mdl"]; !ok { t.Fatalf("expected head feature lock id, got %#v", got[0].LockData) } }