Fix autogen to fail-open on missing asset manifests (#4)

Reviewed-on: #4
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

.gitea/workflows/sync-wrappers.yml

@@ -0,0 +1,50 @@
+# Auto-PR canonical wrapper updates to consumer repos when wrappers/ changes on
+# main. Maintenance automation (not artifact publishing), so it is allowed on a
+# main push under the D7 trigger standard. Requires WRAPPER_SYNC_TOKEN: a bot
+# user's token with content+PR write to the consumer repos (never committed).
+name: sync-wrappers
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'wrappers/crucible.sh'
+      - 'wrappers/crucible.ps1'
+
+jobs:
+  sync:
+    runs-on: nix-docker
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Open sync PRs to consumers
+        env:
+          TOKEN: ${{ secrets.WRAPPER_SYNC_TOKEN }}
+          SERVER: ${{ github.server_url }}
+          SRC_SHA: ${{ github.sha }}
+        run: |
+          nix develop --command bash -c '
+            set -euo pipefail
+            host="$(echo "$SERVER" | sed -E "s#https?://##")"
+            branch="chore/sync-wrappers-$(echo "$SRC_SHA" | cut -c1-12)"
+            grep -vE "^\s*#|^\s*$" wrappers/consumers.txt | while read -r target; do
+              echo "== syncing $target =="
+              work="$(mktemp -d)"
+              git clone "https://oauth2:${TOKEN}@${host}/${target}.git" "$work"
+              cp wrappers/crucible.sh wrappers/crucible.ps1 "$work"/
+              ( cd "$work"
+                git config user.name  "crucible-sync-bot"
+                git config user.email "bot@westgate.pw"
+                if git diff --quiet; then echo "no changes for $target"; exit 0; fi
+                git checkout -b "$branch"
+                git add crucible.sh crucible.ps1
+                git commit -m "chore: sync crucible wrappers from sow-tools@${SRC_SHA}"
+                git push -f origin "$branch"
+                curl -fsS -X POST \
+                  -H "Authorization: token ${TOKEN}" -H "Content-Type: application/json" \
+                  "${SERVER}/api/v1/repos/${target}/pulls" \
+                  -d "{\"head\":\"${branch}\",\"base\":\"main\",\"title\":\"chore: sync crucible wrappers from sow-tools\"}" || true
+              )
+              rm -rf "$work"
+            done
+          '

README.md

@@ -41,6 +41,30 @@ See [`docs/migration-from-nwn-tool.md`](docs/migration-from-nwn-tool.md) for wha
 was done and what remains (the consumer `--manifest/--source/--out` flag contract
 is the open Phase-6 item).
 
+## Quick start (no Nix)
+
+Teammates without Nix don't build anything — they run the bootstrap wrapper,
+which downloads the latest released `crucible` for your OS and runs it:
+
+```bash
+./crucible            # interactive menu (pick a command)
+./crucible module build
+./crucible topdata validate-topdata
+```
+
+Windows (PowerShell):
+
+```powershell
+.\crucible.ps1 module build
+```
+
+The binary is cached under `~/.cache/crucible/<version>/` (`%LOCALAPPDATA%\crucible`
+on Windows); `--repo-local` caches inside the repo instead. Private releases:
+set `CRUCIBLE_TOKEN` or write the token to `~/.config/crucible/token`.
+
+Only the **music** builder needs `ffmpeg`; everything else has zero dependencies.
+If you run a music command without it, Crucible prints a per-OS install hint.
+
 ## Develop
 
 Self-contained (D8) — a host with only Nix can run everything:
@@ -58,12 +82,18 @@ This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
 
 ## CI
 
-PR-first (D7): every check runs on pull requests and on push to `main`.
+PR-first (D7): checks run on pull requests and on push to `main`; the only
+publish event is a `v*` tag (see `sow-docs/runbooks/ci-trigger-standard.md`).
 
-- `test.yml` — vet, test, shellcheck, yamllint, binary smoke.
-- `build-image.yml` — build `registry.westgate.pw/deployment/crucible:<sha>`; publish
-  only on `main`. PRs build but never push. No mutable tags.
-- `release.yml` — tag-gated binary bundles; image re-tag is wired in Phase 6.
+- `test.yml` — vet, test, shellcheck, yamllint, binary smoke (PR + main).
+- `test-image.yml` — build the OCI image to prove it compiles (PR + main, no push).
+- `build-binaries.yml` — cross-build all targets (PR + main); on a `v*` tag, upload
+  the binaries, `SHA256SUMS`, and the wrappers to the Gitea release.
+- `build-image.yml` — on a `v*` tag, build and publish
+  `registry.westgate.pw/deployment/crucible:<sha>`.
+- `publish-image.yml` — manual `workflow_dispatch` break-glass republish.
+- `sync-wrappers.yml` — on a `main` push that touches `wrappers/`, auto-PR the
+  canonical wrappers to the consumer repos in `wrappers/consumers.txt`.
 
 ## Consumers
 

docs/consumer-contract.md

@@ -24,14 +24,17 @@ quoting stays correct.
 
 ## In CI (preferred)
 
-Run the consumer job inside the pinned image and the binaries are on `PATH`:
+**OPERATOR NOTE: Stop. Do not do it this way.**
+**Use the pinned `prod.yml` version. That's what it's there for.**
+
+~~Run the consumer job inside the pinned image and the binaries are on `PATH`:~~
 
 ```yaml
 container:
-  image: registry.westgate.pw/deployment/crucible:<sha>   # pinned, immutable
+  image: registry.westgate.pw/deployment/crucible:<sha> # pinned, immutable
 ```
 
-No host install, no `$HOME` layout, no developer machine assumptions.
+~~No host install, no `$HOME` layout, no developer machine assumptions.~~
 
 ## `NWN_ROOT` rule
 

internal/topdata/autogen.go

@@ -3,6 +3,7 @@ package topdata
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -16,6 +17,19 @@ import (
 
 const autogenManifestCacheMaxAge = time.Hour
 
+// errAutogenManifestUnavailable marks a released autogen manifest that could not
+// be located or fetched (network failure, missing release/asset, empty payload,
+// or an undeterminable sow-assets repo). It is deliberately distinct from an
+// asset that was found but reports a row removed: for an optional consumer this
+// sentinel triggers a fail-open path that *preserves* the consumer's existing
+// pinned lock entries instead of pruning them, so StrRef/row IDs are not
+// reshuffled while the asset source is merely temporarily out of reach.
+var errAutogenManifestUnavailable = errors.New("autogen manifest unavailable")
+
+func unavailableAutogenManifest(err error) error {
+	return fmt.Errorf("%w: %w", errAutogenManifestUnavailable, err)
+}
+
 type autogenManifest struct {
 	ID                string                           `json:"id"`
 	Repo              string                           `json:"repo"`
@@ -61,9 +75,18 @@ func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDatase
 		}
 		manifest, err := resolveAutogenConsumerManifest(p, consumer, progress)
 		if err != nil {
-			if consumer.Optional && isIgnorableOptionalAutogenError(err) {
+			if consumer.Optional && errors.Is(err, errAutogenManifestUnavailable) {
+				// Fail open: the asset manifest is merely unreachable, not
+				// authoritatively empty. Build without its rows, but keep the
+				// consumer's already-pinned lock entries so their IDs are not
+				// freed and reshuffled on a later build once the asset returns.
+				preserved, perr := preserveAutogenConsumerLockEntries(result, consumer)
+				if perr != nil {
+					return nil, perr
+				}
+				result = preserved
 				if progress != nil {
-					progress(fmt.Sprintf("Skipping optional autogen consumer %s: %v", consumer.ID, err))
+					progress(fmt.Sprintf("Autogen consumer %s: released manifest unavailable (%v); preserving existing lock entries, generating no new rows", consumer.ID, err))
 				}
 				continue
 			}
@@ -90,32 +113,115 @@ func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDatase
 
 func autogenConsumerTargetsCollectedDataset(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool {
 	for _, dataset := range collected {
-		switch consumer.Mode {
-		case "parts_rows":
-			if isAutogenEligiblePartsDataset(dataset) {
-				return true
-			}
-		case "accessory_visualeffects":
-			if dataset.Dataset.Name == "visualeffects" {
-				return true
-			}
-		case "cachedmodels_rows":
-			if dataset.Dataset.Name == "cachedmodels" {
-				return true
-			}
+		if autogenConsumerTargetsDataset(dataset, consumer) {
+			return true
 		}
 	}
 	return false
 }
 
-func isIgnorableOptionalAutogenError(err error) bool {
-	if err == nil {
+// preserveAutogenConsumerLockEntries restores the lock keys that an autogen
+// consumer owns from the on-disk lockfile back into the in-memory dataset,
+// reusing this build's already-collected (and pruned) state. It is called only
+// when the consumer's released manifest is unavailable, so the rows themselves
+// are not regenerated; we just keep the key->ID pins alive. Keys are matched
+// against the consumer's own naming policy so unrelated (authored) rows that were
+// legitimately removed are not resurrected.
+func preserveAutogenConsumerLockEntries(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) ([]nativeCollectedDataset, error) {
+	matches := autogenConsumerManagedLockKeyMatcher(consumer)
+	if matches == nil {
+		// No managed-key matcher for this mode: nothing safe to preserve.
+		return collected, nil
+	}
+
+	result := append([]nativeCollectedDataset(nil), collected...)
+	for i, dataset := range result {
+		if !autogenConsumerTargetsDataset(dataset, consumer) {
+			continue
+		}
+		if strings.TrimSpace(dataset.Dataset.LockPath) == "" {
+			continue
+		}
+		onDisk, err := loadLockfile(dataset.Dataset.LockPath)
+		if err != nil {
+			// No readable lockfile means there is nothing pinned to preserve.
+			continue
+		}
+		lockData := dataset.LockData
+		if lockData == nil {
+			lockData = map[string]int{}
+		}
+		restored := 0
+		for key, rowID := range onDisk {
+			if _, present := lockData[key]; present {
+				continue
+			}
+			if !matches(key) {
+				continue
+			}
+			lockData[key] = rowID
+			restored++
+		}
+		if restored == 0 {
+			continue
+		}
+		result[i].LockData = lockData
+		result[i].LockModified = !lockDataEqual(onDisk, lockData)
+	}
+	return result, nil
+}
+
+// autogenConsumerManagedLockKeyMatcher returns a predicate identifying the lock
+// keys an autogen consumer is responsible for, or nil for modes whose keys we
+// cannot scope precisely (in which case preservation is skipped rather than
+// risk resurrecting unrelated keys).
+func autogenConsumerManagedLockKeyMatcher(consumer project.AutogenConsumerConfig) func(string) bool {
+	switch consumer.Mode {
+	case "accessory_visualeffects":
+		policy := resolveHeadVisualeffectsPolicy(consumer, nil)
+		delimiter := policy.Delimiter
+		if delimiter == "" {
+			delimiter = "/"
+		}
+		prefix := "visualeffects:"
+		groups := make(map[string]struct{}, len(policy.Groups))
+		for group, groupPolicy := range policy.Groups {
+			token := applyHeadVisualeffectCase(headVisualeffectGroupToken(group, groupPolicy, policy), policy)
+			if strings.TrimSpace(token) != "" {
+				groups[token] = struct{}{}
+			}
+		}
+		if len(groups) == 0 {
+			return nil
+		}
+		return func(key string) bool {
+			if !strings.HasPrefix(key, prefix) {
+				return false
+			}
+			rest := key[len(prefix):]
+			idx := strings.Index(rest, delimiter)
+			if idx <= 0 {
+				return false
+			}
+			_, ok := groups[rest[:idx]]
+			return ok
+		}
+	default:
+		return nil
+	}
+}
+
+func autogenConsumerTargetsDataset(dataset nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool {
+	switch consumer.Mode {
+	case "parts_rows":
+		return isAutogenEligiblePartsDataset(dataset)
+	case "accessory_visualeffects":
+		return dataset.Dataset.Name == "visualeffects"
+	case "cachedmodels_rows":
+		return dataset.Dataset.Name == "cachedmodels"
+	default:
 		return false
 	}
-	message := err.Error()
-	return strings.Contains(message, "HTTP 404") ||
-		strings.Contains(message, "not found") ||
-		strings.Contains(message, "entries is empty")
 }
 
 func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) {
@@ -366,14 +472,14 @@ func resolveReleasedAutogenManifest(p *project.Project, consumer project.Autogen
 
 	spec, err := deriveSowAssetsRepoSpec(p)
 	if err != nil {
-		return nil, err
+		return nil, unavailableAutogenManifest(err)
 	}
 	manifestURL, err := resolveAutogenManifestAssetURL(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName)
 	if err != nil {
 		if consumer.Mode == "parts_rows" {
-			return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1))
+			return nil, unavailableAutogenManifest(fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)))
 		}
-		return nil, err
+		return nil, unavailableAutogenManifest(err)
 	}
 	if progress != nil {
 		progress(fmt.Sprintf("Fetching released autogen manifest from %s...", manifestURL))
@@ -381,9 +487,9 @@ func resolveReleasedAutogenManifest(p *project.Project, consumer project.Autogen
 	manifest, err := fetchAutogenManifest(manifestURL)
 	if err != nil {
 		if consumer.Mode == "parts_rows" {
-			return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1))
+			return nil, unavailableAutogenManifest(fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)))
 		}
-		return nil, err
+		return nil, unavailableAutogenManifest(err)
 	}
 	if manifest.ID != "" && manifest.ID != consumer.Producer {
 		return nil, fmt.Errorf("autogen manifest %s declared id %q, expected %q", consumer.Manifest.AssetName, manifest.ID, consumer.Producer)

internal/topdata/autogen_preserve_test.go

@@ -0,0 +1,109 @@
+package topdata
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+// When an optional autogen consumer's released manifest cannot be fetched, the
+// build must fail open: it keeps the consumer's already-pinned lock entries
+// (so their IDs are not freed and reshuffled later) and generates no new rows.
+func TestApplyAutogenConsumersPreservesLockEntriesWhenManifestUnavailable(t *testing.T) {
+	root := testProjectRoot(t)
+
+	// 404 for every request → released manifest is unavailable (not empty).
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.NotFound(w, r)
+	}))
+	t.Cleanup(srv.Close)
+	t.Setenv("SOW_ASSETS_SERVER_URL", srv.URL)
+	t.Setenv("SOW_ASSETS_REPO", "ShadowsOverWestgate/sow-assets")
+
+	p := testProject(root)
+	p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
+		{
+			ID:       "accessory_visualeffects",
+			Producer: "accessory_visualeffects",
+			Dataset:  "visualeffects",
+			Mode:     "accessory_visualeffects",
+			Optional: true,
+			Root:     "vfxs",
+			Include:  []string{"head_accessories/**/*.mdl"},
+			Derive:   project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"},
+			Manifest: project.AutogenManifestConfig{ReleaseTag: "head-vfx-manifest-current", AssetName: "sow-accessory-vfx-manifest.json", CacheName: "sow-accessory-vfx-manifest.json"},
+		},
+	}
+
+	// On-disk lock pins an autogen-owned accessory key plus an authored key.
+	lockPath := filepath.Join(root, "visualeffects-lock.json")
+	writeFile(t, lockPath, `{"visualeffects:existing":17,"visualeffects:head_accessories/hat/bandana":10101}`+"\n")
+
+	// Simulate post-prune collected state: the accessory key has already been
+	// dropped from in-memory LockData (as pruneLockDataToActiveRows would do).
+	collected := []nativeCollectedDataset{
+		{
+			Dataset:  nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase, LockPath: lockPath},
+			Columns:  []string{"Label"},
+			Rows:     []map[string]any{{"id": 17, "key": "visualeffects:existing"}},
+			LockData: map[string]int{"visualeffects:existing": 17},
+		},
+	}
+
+	got, err := applyAutogenConsumers(p, collected, nil)
+	if err != nil {
+		t.Fatalf("expected fail-open build, got error: %v", err)
+	}
+	if len(got) != 1 {
+		t.Fatalf("expected one dataset, got %d", len(got))
+	}
+
+	// The pinned accessory id must survive with its exact value (no jumble).
+	if id, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok || id != 10101 {
+		t.Fatalf("expected preserved accessory lock id 10101, got %v (present=%v) lock=%#v", id, ok, got[0].LockData)
+	}
+	// The authored key is untouched.
+	if id, ok := got[0].LockData["visualeffects:existing"]; !ok || id != 17 {
+		t.Fatalf("expected authored key retained at 17, got %#v", got[0].LockData)
+	}
+	// No row is generated for the unavailable accessory entry.
+	for _, row := range got[0].Rows {
+		if key, _ := row["key"].(string); key == "visualeffects:head_accessories/hat/bandana" {
+			t.Fatalf("expected no generated row while manifest unavailable, got %#v", row)
+		}
+	}
+}
+
+// A key that is NOT owned by the consumer (an authored row legitimately removed)
+// must not be resurrected by the preservation path.
+func TestPreserveAutogenConsumerLockEntriesIgnoresUnownedKeys(t *testing.T) {
+	root := testProjectRoot(t)
+	lockPath := filepath.Join(root, "visualeffects-lock.json")
+	writeFile(t, lockPath, `{"visualeffects:retired_authored":3,"visualeffects:head_accessories/hat/bandana":10101}`+"\n")
+
+	consumer := project.AutogenConsumerConfig{
+		ID:      "accessory_visualeffects",
+		Dataset: "visualeffects",
+		Mode:    "accessory_visualeffects",
+	}
+	collected := []nativeCollectedDataset{
+		{
+			Dataset:  nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase, LockPath: lockPath},
+			LockData: map[string]int{},
+		},
+	}
+
+	got, err := preserveAutogenConsumerLockEntries(collected, consumer)
+	if err != nil {
+		t.Fatalf("preserveAutogenConsumerLockEntries failed: %v", err)
+	}
+	if _, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok {
+		t.Fatalf("expected owned accessory key preserved, got %#v", got[0].LockData)
+	}
+	if _, ok := got[0].LockData["visualeffects:retired_authored"]; ok {
+		t.Fatalf("expected unowned authored key NOT resurrected, got %#v", got[0].LockData)
+	}
+}

wrappers/consumers.txt

@@ -0,0 +1,5 @@
+# Repos that carry copies of the canonical Crucible wrappers.
+# One "owner/repo" per line. sync-wrappers.yml opens an update PR to each when
+# wrappers/ changes. Add a repo here AND grant the sync bot write access to it.
+ShadowsOverWestgate/sow-module
+ShadowsOverWestgate/sow-topdata