fix(extract): keep committed palette skeletons out of stale cleanup (#96)

Extract skips writing *palcus.itp back to source (build projects them from blueprints, #50), but the skip branch never marked the skeleton target as desired, so cleanupStaleFiles deleted the committed source/palettes/*.itp.json on every extract — shipping modules with empty custom palettes. Fix marks the skipped target desired; regression test builds a module with a committed skeleton and asserts extract leaves it in place.

🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #96

Co-authored-by: vickydotbat <vickydotbat@tutamail.com>

docs/command-surface.md

@@ -97,17 +97,22 @@ broken — is skipped by every later run forever and no backfill repairs it. Wit
 and replaced when it does not match. It costs a full GET per existing blob, so
 it is a repair pass, not the default.
 
-**After a repair, purge the pull zone before believing `verify`.** A repair is
+**After a repair, `verify` is what tells you which keys to purge.** A repair is
 the one thing that makes a key serve different bytes than it did before, and the
 edge caches these objects for 30 days precisely because that normally cannot
-happen. The two commands therefore look at different copies on purpose: `emit
+happen. The two commands look at different copies on purpose: `emit --verify`
---verify` repairs the **origin**, `verify` reads the **edge**, and in between a
+repairs the **origin**, `verify` reads the **edge**. So a `verify` run straight
-warm PoP still answers with the old bytes while a cold one answers with the new.
+after a repair is not a verdict — it is a survey, and every blob it still calls
-Until the zone is purged `verify`'s verdict is per-PoP and settles nothing — a
+bad is one the edge is serving stale. Purge exactly those, then re-run it; only
-pass is not proof, and a failure is not the repair having failed. The purge is
+that second run is the verdict.
-one call against the pull zone; it belongs in the repair procedure rather than
+
-in `emit`, which holds a storage credential and no CDN one (sow-tools#89, and
+Purging the keys `verify` names beats purging the zone, because the edge only
-the procedure itself is in sow-platform's NWSync runbook).
+ever cached what somebody actually fetched: the 2026-08-01 repair rewrote 2,603
+blobs at the origin and left 8 stale at the edge. The purge belongs in the
+repair procedure rather than in `emit`, which reports how many blobs it wrote
+and never which ones — so it could not target one even with a CDN credential,
+which it deliberately does not hold (#89; the procedure itself is in
+sow-platform's NWSync runbook).
 
 `emit` uploads blobs first and the index last, so the presence of an index is
 the publication marker: an artifact whose emit died halfway leaves real blobs in

internal/nwsync/run.go

@@ -67,8 +67,9 @@ check on a published blob upstream of a player's client.
 --verify makes emit hash what it would otherwise skip. emit normally treats a
 blob's presence as proof of its contents, so without this an object written
 truncated, or written by an emitter since found broken, is skipped forever.
---verify repairs the storage zone, while verify reads the edge in front of it,
+--verify repairs the storage zone, while verify reads the edge in front of it.
-so purge the pull zone after a repair or verify answers differently per PoP.
+So a verify run right after a repair is a survey, not a verdict: it names the
+keys the edge still serves stale. Purge those, then run it again.
 
 --out DIR writes to a local repository tree instead of uploading, which is the
 conformance path against upstream nwn_nwsync_write. Without it, the zone comes

internal/pipeline/extract.go

@@ -126,8 +126,14 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
 		}
 		// *palcus.itp are Toolset-generated palette projections; the module
 		// build regenerates them from source blueprints, so extraction never
-		// writes them back into source.
+		// writes them back into source. The committed skeleton still has to
+		// survive stale cleanup, so mark its target as desired.
 		if ext == "itp" && isPaletteProjectionResref(resource.Name) {
+			effective := p.EffectiveConfig()
+			target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(ext), strings.ToLower(resource.Name)+"."+ext+".json")
+			if err == nil {
+				desired[target] = struct{}{}
+			}
 			skippedCount++
 			continue
 		}

internal/pipeline/pipeline_test.go

@@ -3312,6 +3312,88 @@ func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
 	}
 }
 
+func TestExtractKeepsPaletteSkeletonsThroughStaleCleanup(t *testing.T) {
+	root := t.TempDir()
+	mustMkdir(t, filepath.Join(root, "src", "module"))
+	mustMkdir(t, filepath.Join(root, "src", "palettes"))
+	mustMkdir(t, filepath.Join(root, "assets"))
+	mustMkdir(t, filepath.Join(root, "build"))
+
+	mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
+  "module": {
+    "name": "Test Module",
+    "resref": "testmod"
+  },
+  "paths": {
+    "source": "src",
+    "assets": "assets",
+    "build": "build"
+  }
+}
+`)
+
+	mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
+  "file_type": "IFO ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 0,
+    "fields": [
+      {
+        "label": "Mod_Name",
+        "type": "CExoString",
+        "value": "Original Module"
+      }
+    ]
+  }
+}
+`)
+	skeletonPath := filepath.Join(root, "src", "palettes", "creaturepalcus.itp.json")
+	mustWriteFile(t, skeletonPath, `{
+  "file_type": "ITP ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 0,
+    "fields": [
+      {
+        "label": "MAIN",
+        "type": "List",
+        "value": []
+      }
+    ]
+  }
+}
+`)
+
+	p, err := project.Load(root)
+	if err != nil {
+		t.Fatalf("load project: %v", err)
+	}
+	if err := p.ValidateLayout(); err != nil {
+		t.Fatalf("validate layout: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+
+	if _, err := BuildModule(p); err != nil {
+		t.Fatalf("build module: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("rescan before extract: %v", err)
+	}
+
+	result, err := Extract(p)
+	if err != nil {
+		t.Fatalf("extract: %v", err)
+	}
+	if result.Removed != 0 {
+		t.Fatalf("expected no removed files, got %d", result.Removed)
+	}
+	if _, err := os.Stat(skeletonPath); err != nil {
+		t.Fatalf("expected palette skeleton to survive extract, stat err=%v", err)
+	}
+}
+
 func TestExtractMergesConfiguredGFFJSONFieldsAndLists(t *testing.T) {
 	root := t.TempDir()
 	mustMkdir(t, filepath.Join(root, "src", "module"))

internal/project/project.go

@@ -560,8 +560,18 @@ func (p *Project) ValidateLayout() error {
 	if strings.TrimSpace(p.Config.Module.ResRef) == "" {
 		failures = append(failures, errors.New("module.resref is required"))
 	}
-	if len(p.Config.Module.ResRef) > 16 {
+	// module.resref names the built .mod FILE, so the 16-byte resref limit does not
-		failures = append(failures, fmt.Errorf("module.resref %q exceeds 16 characters", p.Config.Module.ResRef))
+	// apply to it — NWN:EE module file names are routinely longer. It is validated as
+	// a file name instead. The limit still binds when the same value has to be a real
+	// resref: with no haks configured, an asset project names its single generated HAK
+	// after it, and a HAK name is a resref the engine loads.
+	if err := validateOutputFileName("module.resref", p.Config.Module.ResRef+".mod", ".mod"); err != nil {
+		failures = append(failures, err)
+	}
+	if len(p.Config.Module.ResRef) > 16 && strings.TrimSpace(p.Config.Paths.Assets) != "" && len(p.Config.HAKs) == 0 {
+		failures = append(failures, fmt.Errorf(
+			"module.resref %q exceeds 16 characters and would name this project's generated HAK; configure haks[] with a shorter name",
+			p.Config.Module.ResRef))
 	}
 	if strings.TrimSpace(p.Config.Paths.Source) == "" && strings.TrimSpace(p.Config.Paths.Assets) == "" && !p.HasTopData() {
 		failures = append(failures, errors.New("at least one of paths.source, paths.assets, or topdata.source is required"))

internal/project/project_test.go

@@ -1093,6 +1093,78 @@ func TestValidateLayoutAllowsMissingAssetsDir(t *testing.T) {
 	}
 }
 
+// module.resref names the built .mod FILE, not a resource inside an archive, so the
+// 16-byte resref limit does not apply to it. NWN:EE module file names are commonly
+// longer (ShadowsOverWestgate.mod is 19). The limit still binds everywhere a resref
+// really is a resref — see TestValidateLayoutRejectsLongResRefWhenItNamesAHAK.
+func TestValidateLayoutAllowsLongModuleResRef(t *testing.T) {
+	root := t.TempDir()
+	mkdirAll(t, filepath.Join(root, "src"))
+	mkdirAll(t, filepath.Join(root, "build"))
+
+	proj := &Project{
+		Root: root,
+		Config: Config{
+			Module: ModuleConfig{Name: "Shadows Over Westgate", ResRef: "ShadowsOverWestgate"},
+			Paths:  PathConfig{Source: "src", Build: "build"},
+		},
+	}
+
+	if err := proj.ValidateLayout(); err != nil {
+		t.Fatalf("ValidateLayout rejected a 19-character module file name: %v", err)
+	}
+	if got, want := filepath.Base(proj.ModuleArchivePath()), "ShadowsOverWestgate.mod"; got != want {
+		t.Fatalf("ModuleArchivePath() = %q, want %q", got, want)
+	}
+}
+
+// A module.resref that is not a usable file name is still rejected.
+func TestValidateLayoutRejectsModuleResRefThatIsAPath(t *testing.T) {
+	root := t.TempDir()
+	mkdirAll(t, filepath.Join(root, "src"))
+
+	proj := &Project{
+		Root: root,
+		Config: Config{
+			Module: ModuleConfig{Name: "Test", ResRef: "../escape/mod"},
+			Paths:  PathConfig{Source: "src", Build: "build"},
+		},
+	}
+
+	err := proj.ValidateLayout()
+	if err == nil {
+		t.Fatal("ValidateLayout accepted a module.resref containing a path")
+	}
+	if !strings.Contains(err.Error(), "module.resref") {
+		t.Fatalf("error does not name the offending field: %v", err)
+	}
+}
+
+// When a project declares no haks, the module resref becomes the name of the single
+// generated HAK — and a HAK name IS a resref the engine loads. The limit applies
+// there, so a long name is only allowed for projects that build no HAKs.
+func TestValidateLayoutRejectsLongResRefWhenItNamesAHAK(t *testing.T) {
+	root := t.TempDir()
+	mkdirAll(t, filepath.Join(root, "src"))
+	mkdirAll(t, filepath.Join(root, "assets"))
+
+	proj := &Project{
+		Root: root,
+		Config: Config{
+			Module: ModuleConfig{Name: "Shadows Over Westgate", ResRef: "ShadowsOverWestgate"},
+			Paths:  PathConfig{Source: "src", Assets: "assets", Build: "build"},
+		},
+	}
+
+	err := proj.ValidateLayout()
+	if err == nil {
+		t.Fatal("ValidateLayout accepted a 19-character name for a generated HAK")
+	}
+	if !strings.Contains(err.Error(), "16") {
+		t.Fatalf("error does not explain the resref limit: %v", err)
+	}
+}
+
 // paths.build is an OUTPUT dir the builder creates (MkdirAll) before writing, so
 // a bare clone with no build dir yet must still validate/build with no pre-step
 // (R2/parity). Only a build path that exists but is not a directory is an error.

internal/topdata/generated_assets.go

@@ -98,8 +98,9 @@ func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopDat
 	}
 
 	results := make([]Generated2DAAsset, 0, len(collected))
+	racialFeatRules := racialUsableFeatRules(collected)
 	for _, dataset := range collected {
-		compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil, project.TopDataClassFeatInjectionConfig{}, nil)
+		compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil, project.TopDataClassFeatInjectionConfig{}, racialFeatRules, nil)
 		if err != nil {
 			return nil, err
 		}

internal/topdata/native.go

@@ -530,6 +530,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
 	groupStats := nativeCompileGroupStats(collected)
 	currentGroup := ""
 	sidecars := newNativeSidecarCollector()
+	racialFeatRules := racialUsableFeatRules(collected)
 	for _, dataset := range collected {
 		group := nativeCompileGroup(dataset.Dataset.Name)
 		if group != currentGroup {
@@ -542,7 +543,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
 				stats.SourceFragments,
 			))
 		}
-		compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler, p.EffectiveConfig().TopData.ClassFeatInjections, sidecars)
+		compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler, p.EffectiveConfig().TopData.ClassFeatInjections, racialFeatRules, sidecars)
 		if err != nil {
 			return BuildResult{}, err
 		}
@@ -3996,13 +3997,13 @@ func normalizeGlobalReferenceID(value string) string {
 	return parts[0] + ":" + strings.ReplaceAll(parts[1], "_", "")
 }
 
-func resolveNativeDataset(dataset nativeCollectedDataset, keyToID map[string]int, globalRowByKey map[string]map[string]any, tableRegistry resolvedTableRegistry, compiler *tlkCompiler, classFeatInjections project.TopDataClassFeatInjectionConfig, sidecars *nativeSidecarCollector) (map[string]any, error) {
+func resolveNativeDataset(dataset nativeCollectedDataset, keyToID map[string]int, globalRowByKey map[string]map[string]any, tableRegistry resolvedTableRegistry, compiler *tlkCompiler, classFeatInjections project.TopDataClassFeatInjectionConfig, racialFeatRules []project.TopDataClassFeatGlobalRule, sidecars *nativeSidecarCollector) (map[string]any, error) {
 	rows := dataset.Rows
 	if strings.HasPrefix(filepath.ToSlash(dataset.Dataset.Name), "classes/feats/") {
 		classKey := "classes:" + dataset.Dataset.Name[strings.LastIndex(dataset.Dataset.Name, "/")+1:]
 		featSuccessors := buildFeatSuccessorsIndex(globalRowByKey, keyToID)
 		classSkills := buildClassSkillsIndex(tableRegistry, classKey)
-		expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey, classKey, classFeatInjections, !dataset.Dataset.HasGlobalInjections)
+		expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey, classKey, classFeatInjections, racialFeatRules, !dataset.Dataset.HasGlobalInjections)
 		if err != nil {
 			return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
 		}
@@ -4051,12 +4052,18 @@ var (
 	}
 )
 
-func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any, classKey string, classFeatInjections project.TopDataClassFeatInjectionConfig, useConfiguredInjections bool) ([]map[string]any, error) {
+func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any, classKey string, classFeatInjections project.TopDataClassFeatInjectionConfig, racialFeatRules []project.TopDataClassFeatGlobalRule, useConfiguredInjections bool) ([]map[string]any, error) {
 
 	globalRules, classSkillRules := []project.TopDataClassFeatGlobalRule{}, []project.TopDataClassFeatMasterfeatRule{}
 	if useConfiguredInjections {
 		globalRules, classSkillRules = effectiveClassFeatInjectionRules(classFeatInjections)
 	}
+	// Racial usable-feat rows are generated from the race feats tables, not the
+	// hand-authored class-feat injections, so they apply to every class table
+	// regardless of whether that dataset carries its own global.json (which is
+	// what gates useConfiguredInjections). Deduped below against rows already
+	// present, so a leftover hand row in global.json is a no-op, not a double.
+	globalRules = append(globalRules, racialFeatRules...)
 	injected := make([]map[string]any, 0, len(globalRules)+len(classSkillRules))
 	presentRefIDs := make(map[string]struct{}, len(rows))
 	for _, row := range rows {
@@ -4134,6 +4141,54 @@ func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByK
 	return combined, nil
 }
 
+// racialUsableFeatRules builds one class-feat injection rule per feat marked
+// UsableFeat=1 in any race feats table (race_feat_*.2da). Racial feats are
+// granted by race, never by a class, so an activatable one needs a menu-only
+// cls_feat row to reach the client radial: List=3 keeps it off every level-up
+// selection list, GrantedOnLevel=99 sits above the level cap so no class ever
+// actually grants it, and OnMenu=1 renders the button once the creature holds
+// the feat (possession comes from chargen / the login racial-feat sync). This
+// replaces the hand-maintained racial rows in classes/feats/global.json - mark
+// UsableFeat in the race table and the radial row follows automatically.
+func racialUsableFeatRules(collected []nativeCollectedDataset) []project.TopDataClassFeatGlobalRule {
+	seen := map[string]struct{}{}
+	rules := []project.TopDataClassFeatGlobalRule{}
+	for _, ds := range collected {
+		if !strings.HasPrefix(ds.Dataset.OutputName, "race_feat_") || !strings.HasSuffix(ds.Dataset.OutputName, ".2da") {
+			continue
+		}
+		for _, row := range ds.Rows {
+			if usable, err := asInt(fieldValue(row, "UsableFeat")); err != nil || usable != 1 {
+				continue
+			}
+			featRef, ok := row["FeatIndex"].(map[string]any)
+			if !ok {
+				continue
+			}
+			featID, _ := featRef["id"].(string)
+			if featID == "" {
+				continue
+			}
+			if _, dup := seen[featID]; dup {
+				continue
+			}
+			seen[featID] = struct{}{}
+			rules = append(rules, project.TopDataClassFeatGlobalRule{
+				Feat:           featID,
+				List:           "3",
+				GrantedOnLevel: "99",
+				OnMenu:         "1",
+			})
+		}
+	}
+	// Discovery order across files and the dedup map are both unordered; sort so
+	// the injected rows (and the resulting 2DA row numbering) are deterministic.
+	slices.SortFunc(rules, func(a, b project.TopDataClassFeatGlobalRule) int {
+		return strings.Compare(a.Feat, b.Feat)
+	})
+	return rules
+}
+
 func effectiveClassFeatInjectionRules(config project.TopDataClassFeatInjectionConfig) ([]project.TopDataClassFeatGlobalRule, []project.TopDataClassFeatMasterfeatRule) {
 	if len(config.GlobalFeats) == 0 && len(config.ClassSkillMasterfeats) == 0 {
 		return defaultClassFeatGlobalRules, defaultClassFeatClassSkillMasterfeatRules

internal/topdata/racial_feat_rules_test.go

@@ -0,0 +1,54 @@
+package topdata
+
+import (
+	"reflect"
+	"testing"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+func raceFeatDataset(output string, rows ...map[string]any) nativeCollectedDataset {
+	return nativeCollectedDataset{
+		Dataset: nativeDataset{OutputName: output},
+		Rows:    rows,
+	}
+}
+
+func usableRow(featID string, usable any) map[string]any {
+	row := map[string]any{"FeatIndex": map[string]any{"id": featID}}
+	if usable != nil {
+		row["UsableFeat"] = usable
+	}
+	return row
+}
+
+func TestRacialUsableFeatRules(t *testing.T) {
+	collected := []nativeCollectedDataset{
+		raceFeatDataset("race_feat_ddrw.2da",
+			usableRow("feat:keen_sense", nil),      // passive, no UsableFeat -> skipped
+			usableRow("feat:darkvision", 1),        // usable
+			usableRow("feat:use_poison", 0),        // explicitly not usable -> skipped
+			usableRow("feat:drow/faerie_fire", "1"),// usable, string form
+		),
+		raceFeatDataset("race_feat_tief.2da",
+			usableRow("feat:darkvision", 1), // duplicate across races -> collapses to one
+			usableRow("feat:tiefling/darkness", 1),
+		),
+		raceFeatDataset("feat.2da", // not a race feats table -> ignored entirely
+			usableRow("feat:power_attack", 1),
+		),
+	}
+
+	got := racialUsableFeatRules(collected)
+
+	want := []project.TopDataClassFeatGlobalRule{
+		{Feat: "feat:darkvision", List: "3", GrantedOnLevel: "99", OnMenu: "1"},
+		{Feat: "feat:drow/faerie_fire", List: "3", GrantedOnLevel: "99", OnMenu: "1"},
+		{Feat: "feat:tiefling/darkness", List: "3", GrantedOnLevel: "99", OnMenu: "1"},
+	}
+
+	// Slice is sorted by Feat, so order is deterministic.
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("rules: got %+v, want %+v", got, want)
+	}
+}

internal/topdata/topdata_test.go

@@ -1523,6 +1523,7 @@ func TestResolveNativeDatasetPreservesScalarTableReferenceBehavior(t *testing.T)
 		nil,
 		project.TopDataClassFeatInjectionConfig{},
 		nil,
+		nil,
 	)
 	if err != nil {
 		t.Fatalf("resolveNativeDataset failed: %v", err)