Generate palette projections at module build (#50)

*palcus.itp files are Toolset-derived snapshots (category skeleton + blueprint descriptors). This makes Crucible the generator: module builds strip stale descriptors from the committed skeletons and inject descriptors computed from the source-tree blueprints (NAME/STRREF + RESREF, plus CR/FACTION for creatures, faction names resolved via repute.fac). Compare applies the same projection; extract never writes *palcus.itp back into source, so local Toolset palette mess stays local.

Companion PR in sow-module strips the committed palcus files down to skeletons.

🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #50

Co-authored-by: vickydotbat <vickydotbat@tutamail.com>

docs/superpowers/specs/2026-07-12-crucible-assets-builder-design.md

@@ -126,8 +126,9 @@ exactly as the reference does:
   - Beamdog install dirs.
   - `--nwn <install>` overrides (path to the install root or directly to the
     binary).
-- **Headless.** The engine is a GUI binary. If there is no `DISPLAY` and
-  `xvfb-run` is present, wrap the call:
+- **Headless.** The engine is a GUI binary. Require `xvfb-run` and wrap the call
+  regardless of the caller's `DISPLAY` so compilation never opens the client
+  UI; fail before invoking the engine when `xvfb-run` is unavailable:
   `xvfb-run -a --server-args=-screen 0 1024x768x24 nwmain-linux compilemodel <stem>`.
 - **Per model (one at a time — the engine's `development/` and `modelcompiler/`
   folders are flat and single-slot):**

internal/assets/compile.go

@@ -40,13 +40,13 @@ func runCompile(args []string, stdout, stderr io.Writer, getenv func(string) str
 	}
 	binDir := filepath.Dir(nwmain)
 
-	// Headless wrap: no DISPLAY + xvfb-run present -> run under a virtual X.
-	var wrap []string
-	if getenv("DISPLAY") == "" {
-		if xvfb := look("xvfb-run"); xvfb != "" {
-			wrap = []string{xvfb, "-a", "--server-args=-screen 0 1024x768x24"}
-		}
+	// Always use a virtual X so compilation never opens the client UI.
+	xvfb := look("xvfb-run")
+	if xvfb == "" {
+		fmt.Fprintln(stderr, "assets compile: xvfb-run not found — install it to run the NWN model compiler headlessly")
+		return exitTool
 	}
+	wrap := []string{xvfb, "-a", "--server-args=-screen 0 1024x768x24"}
 
 	files, err := walk(dirs, mdlExt, !*nonRecursive)
 	if err != nil {

internal/assets/compile_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -23,13 +24,19 @@ func TestCompileDrivesEngineAndReplacesInPlace(t *testing.T) {
 	if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
 		t.Fatal(err)
 	}
+	xvfbDir := t.TempDir()
+	xvfb := filepath.Join(xvfbDir, "xvfb-run")
+	if err := os.WriteFile(xvfb, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", xvfbDir+string(os.PathListSeparator)+os.Getenv("PATH"))
 
 	getenv := func(k string) string {
 		switch k {
 		case "HOME":
 			return home
 		case "DISPLAY":
-			return ":0" // pretend a display exists so no xvfb wrap is needed
+			return ":0" // a desktop display must not make compilation interactive
 		}
 		return ""
 	}
@@ -38,7 +45,11 @@ func TestCompileDrivesEngineAndReplacesInPlace(t *testing.T) {
 	orig := runner
 	defer func() { runner = orig }()
 	runner = func(dir string, env []string, name string, args ...string) ([]byte, error) {
-		// args: compilemodel <stem>
+		if name != xvfb || len(args) != 5 || args[0] != "-a" ||
+			args[1] != "--server-args=-screen 0 1024x768x24" || args[2] != nwmain ||
+			args[3] != "compilemodel" {
+			t.Fatalf("engine command = %q %q, want xvfb-run wrapping nwmain", name, args)
+		}
 		stem := args[len(args)-1]
 		compiled := filepath.Join(mc, stem+".mdl")
 		return nil, os.WriteFile(compiled, []byte("\x00\x00compiled"), 0o644)
@@ -77,6 +88,11 @@ func TestCompileAbortsOnNameMismatch(t *testing.T) {
 	if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
 		t.Fatal(err)
 	}
+	xvfbDir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(xvfbDir, "xvfb-run"), []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", xvfbDir+string(os.PathListSeparator)+os.Getenv("PATH"))
 	getenv := func(k string) string {
 		if k == "HOME" {
 			return home
@@ -108,3 +124,48 @@ func TestCompileAbortsOnNameMismatch(t *testing.T) {
 		t.Fatal("engine should not run for a name-mismatched model")
 	}
 }
+
+func TestCompileFailsClosedWithoutXvfb(t *testing.T) {
+	home := t.TempDir()
+	userData := filepath.Join(home, ".local", "share", "Neverwinter Nights")
+	for _, d := range []string{filepath.Join(userData, "development"), filepath.Join(userData, "modelcompiler")} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	nwmain := filepath.Join(t.TempDir(), "nwmain-linux")
+	if err := os.WriteFile(nwmain, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", t.TempDir())
+
+	getenv := func(k string) string {
+		if k == "HOME" {
+			return home
+		}
+		return ""
+	}
+	orig := runner
+	defer func() { runner = orig }()
+	engineCalled := false
+	runner = func(string, []string, string, ...string) ([]byte, error) {
+		engineCalled = true
+		return nil, nil
+	}
+
+	srcDir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(srcDir, "foo.mdl"),
+		[]byte("newmodel foo\nbeginmodelgeom foo\n  node dummy foo\n    parent null\n  endnode\nendmodelgeom foo\ndonemodel foo\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := runCompile([]string{"--nwn", nwmain, srcDir}, &stdout, &stderr, getenv); code != exitTool {
+		t.Fatalf("compile exit = %d, want %d\n%s", code, exitTool, stderr.String())
+	}
+	if engineCalled {
+		t.Fatal("engine must not run without xvfb-run")
+	}
+	if !strings.Contains(stderr.String(), "xvfb-run") {
+		t.Fatalf("missing actionable xvfb-run error: %s", stderr.String())
+	}
+}

internal/pipeline/build.go

@@ -593,9 +593,14 @@ func envBool(name string) bool {
 func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) {
 	var moduleResources []erf.Resource
 
+	palettes, err := collectPaletteDescriptors(p)
+	if err != nil {
+		return nil, err
+	}
+
 	for _, rel := range p.Inventory.SourceFiles {
 		abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
-		resource, err := resourceFromJSON(abs, moduleHakOrder)
+		resource, err := resourceFromJSON(abs, moduleHakOrder, palettes)
 		if err != nil {
 			return nil, err
 		}
@@ -1030,7 +1035,7 @@ func compareResourceKeys(a, b erf.Resource) int {
 	return 0
 }
 
-func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) {
+func resourceFromJSON(path string, moduleHakOrder []string, palettes paletteProjection) (erf.Resource, error) {
 	name, extension, err := splitSourceName(path)
 	if err != nil {
 		return erf.Resource{}, err
@@ -1059,6 +1064,9 @@ func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error
 	if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 {
 		setModuleHAKList(&document, moduleHakOrder)
 	}
+	if extension == ".itp" && isPaletteProjectionResref(name) {
+		projectPaletteDocument(&document, palettes[strings.ToLower(name)])
+	}
 
 	var buf bytes.Buffer
 	if err := gff.Write(&buf, document); err != nil {

internal/pipeline/compare.go

@@ -117,6 +117,10 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
 	if err != nil {
 		return nil, err
 	}
+	palettes, err := collectPaletteDescriptors(p)
+	if err != nil {
+		return nil, err
+	}
 
 	for _, rel := range p.Inventory.SourceFiles {
 		abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
@@ -136,6 +140,9 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro
 		if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
 			setModuleHAKList(&document, moduleHakOrder)
 		}
+		if extension == ".itp" && isPaletteProjectionResref(name) {
+			projectPaletteDocument(&document, palettes[strings.ToLower(name)])
+		}
 
 		canonical, err := json.Marshal(document)
 		if err != nil {

internal/pipeline/extract.go

@@ -124,6 +124,13 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
 			skippedCount++
 			continue
 		}
+		// *palcus.itp are Toolset-generated palette projections; the module
+		// build regenerates them from source blueprints, so extraction never
+		// writes them back into source.
+		if ext == "itp" && isPaletteProjectionResref(resource.Name) {
+			skippedCount++
+			continue
+		}
 
 		target, data, err := extractedFile(p, resource, ext)
 		if err != nil {

internal/pipeline/palette.go

@@ -0,0 +1,298 @@
+package pipeline
+
+// Custom palette projections (*palcus.itp) are generated artifacts: the Toolset
+// derives them from the category frameworks plus the blueprints present in the
+// module. Crucible reproduces that projection deterministically at build time so
+// the module source only carries descriptor-free category skeletons and the
+// blueprint files themselves. Extract never writes palcus files back.
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+const noStrref = 0xFFFFFFFF
+
+// hiddenPaletteID suppresses a blueprint from the Custom palette (engine
+// convention; see docs in sow-module's palette research notes).
+const hiddenPaletteID = 255
+
+var paletteFamilyByExtension = map[string]string{
+	".utc": "creaturepalcus",
+	".utd": "doorpalcus",
+	".ute": "encounterpalcus",
+	".uti": "itempalcus",
+	".utm": "storepalcus",
+	".utp": "placeablepalcus",
+	".uts": "soundpalcus",
+	".utt": "triggerpalcus",
+	".utw": "waypointpalcus",
+}
+
+type paletteDescriptor struct {
+	name     string
+	strref   uint32
+	resref   string
+	creature bool
+	cr       float32
+	faction  string
+}
+
+func (d paletteDescriptor) sortKey() string {
+	if d.strref != noStrref || d.name == "" {
+		return strings.ToLower(d.resref)
+	}
+	return strings.ToLower(d.name)
+}
+
+// palette resref (e.g. "itempalcus") -> terminal category ID -> descriptors.
+type paletteProjection map[string]map[uint8][]paletteDescriptor
+
+func isPaletteProjectionResref(name string) bool {
+	return strings.HasSuffix(strings.ToLower(name), "palcus")
+}
+
+func collectPaletteDescriptors(p *project.Project) (paletteProjection, error) {
+	projection := paletteProjection{}
+	factions, err := loadFactionNames(p)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, rel := range p.Inventory.SourceFiles {
+		abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
+		name, extension, err := splitSourceName(abs)
+		if err != nil {
+			return nil, err
+		}
+		family, ok := paletteFamilyByExtension[extension]
+		if !ok {
+			continue
+		}
+
+		raw, err := os.ReadFile(abs)
+		if err != nil {
+			return nil, fmt.Errorf("read %s: %w", abs, err)
+		}
+		var document gff.Document
+		if err := json.Unmarshal(raw, &document); err != nil {
+			return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
+		}
+
+		descriptor, paletteID, ok := blueprintDescriptor(document.Root, name, extension, factions)
+		if !ok {
+			continue
+		}
+		if projection[family] == nil {
+			projection[family] = map[uint8][]paletteDescriptor{}
+		}
+		projection[family][paletteID] = append(projection[family][paletteID], descriptor)
+	}
+
+	for _, byID := range projection {
+		for _, descriptors := range byID {
+			sort.SliceStable(descriptors, func(i, j int) bool {
+				a, b := descriptors[i], descriptors[j]
+				if a.sortKey() != b.sortKey() {
+					return a.sortKey() < b.sortKey()
+				}
+				return a.resref < b.resref
+			})
+		}
+	}
+	return projection, nil
+}
+
+func blueprintDescriptor(root gff.Struct, fileName, extension string, factions []string) (paletteDescriptor, uint8, bool) {
+	descriptor := paletteDescriptor{
+		strref:   noStrref,
+		resref:   strings.ToLower(fileName),
+		creature: extension == ".utc",
+	}
+	paletteID := -1
+
+	for _, field := range root.Fields {
+		switch field.Label {
+		case "PaletteID":
+			if v, ok := field.Value.(gff.ByteValue); ok {
+				paletteID = int(v)
+			}
+		case "TemplateResRef":
+			if v, ok := field.Value.(gff.ResRefValue); ok && v != "" {
+				descriptor.resref = strings.ToLower(string(v))
+			}
+		case "LocalizedName", "LocName", "FirstName":
+			if v, ok := field.Value.(gff.LocString); ok {
+				name, strref := locStringLabel(v)
+				if field.Label == "FirstName" {
+					descriptor.name = strings.TrimSpace(descriptor.name + " " + name)
+					if descriptor.strref == noStrref {
+						descriptor.strref = strref
+					}
+				} else {
+					descriptor.name = name
+					descriptor.strref = strref
+				}
+			}
+		case "LastName":
+			if v, ok := field.Value.(gff.LocString); ok {
+				name, _ := locStringLabel(v)
+				descriptor.name = strings.TrimSpace(descriptor.name + " " + name)
+			}
+		case "ChallengeRating":
+			if v, ok := field.Value.(gff.FloatValue); ok {
+				descriptor.cr = float32(v)
+			}
+		case "FactionID":
+			if v, ok := field.Value.(gff.WordValue); ok && int(v) < len(factions) {
+				descriptor.faction = factions[v]
+			}
+		}
+	}
+
+	if paletteID < 0 || paletteID == hiddenPaletteID {
+		return paletteDescriptor{}, 0, false
+	}
+	return descriptor, uint8(paletteID), true
+}
+
+func locStringLabel(value gff.LocString) (string, uint32) {
+	if value.StringRef != noStrref {
+		return "", value.StringRef
+	}
+	for _, entry := range value.Entries {
+		if entry.Value != "" {
+			return entry.Value, noStrref
+		}
+	}
+	return "", noStrref
+}
+
+func loadFactionNames(p *project.Project) ([]string, error) {
+	for _, rel := range p.Inventory.SourceFiles {
+		if !strings.HasSuffix(strings.ToLower(rel), "repute.fac.json") {
+			continue
+		}
+		abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
+		raw, err := os.ReadFile(abs)
+		if err != nil {
+			return nil, fmt.Errorf("read %s: %w", abs, err)
+		}
+		var document gff.Document
+		if err := json.Unmarshal(raw, &document); err != nil {
+			return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
+		}
+		var names []string
+		for _, field := range document.Root.Fields {
+			if field.Label != "FactionList" {
+				continue
+			}
+			list, ok := field.Value.(gff.ListValue)
+			if !ok {
+				continue
+			}
+			for _, faction := range list {
+				name := ""
+				for _, f := range faction.Fields {
+					if f.Label == "FactionName" {
+						if v, ok := f.Value.(gff.StringValue); ok {
+							name = string(v)
+						}
+					}
+				}
+				names = append(names, name)
+			}
+		}
+		return names, nil
+	}
+	return nil, nil
+}
+
+// projectPaletteDocument strips every blueprint descriptor from the palette
+// tree and re-inserts the descriptors derived from module source. Category
+// structure (branches, terminal IDs, labels) passes through untouched.
+func projectPaletteDocument(document *gff.Document, byID map[uint8][]paletteDescriptor) {
+	for i, field := range document.Root.Fields {
+		if field.Label != "MAIN" {
+			continue
+		}
+		if list, ok := field.Value.(gff.ListValue); ok {
+			document.Root.Fields[i] = gff.NewField("MAIN", projectPaletteNodes(list, byID))
+		}
+	}
+}
+
+func projectPaletteNodes(nodes gff.ListValue, byID map[uint8][]paletteDescriptor) gff.ListValue {
+	out := make(gff.ListValue, 0, len(nodes))
+	for _, node := range nodes {
+		if structHasField(node, "RESREF") {
+			continue // blueprint descriptor — regenerated below
+		}
+
+		terminalID := -1
+		fields := make([]gff.Field, 0, len(node.Fields))
+		for _, field := range node.Fields {
+			if field.Label == "LIST" {
+				continue // rebuilt for terminals, recursed for branches
+			}
+			if field.Label == "ID" {
+				if v, ok := field.Value.(gff.ByteValue); ok {
+					terminalID = int(v)
+				}
+			}
+			fields = append(fields, field)
+		}
+
+		switch {
+		case terminalID >= 0:
+			if descriptors := byID[uint8(terminalID)]; len(descriptors) > 0 {
+				fields = append(fields, gff.NewField("LIST", descriptorList(descriptors)))
+			}
+		default:
+			for _, field := range node.Fields {
+				if field.Label == "LIST" {
+					if list, ok := field.Value.(gff.ListValue); ok {
+						fields = append(fields, gff.NewField("LIST", projectPaletteNodes(list, byID)))
+					}
+				}
+			}
+		}
+		out = append(out, gff.Struct{Type: node.Type, Fields: fields})
+	}
+	return out
+}
+
+func descriptorList(descriptors []paletteDescriptor) gff.ListValue {
+	list := make(gff.ListValue, 0, len(descriptors))
+	for _, d := range descriptors {
+		fields := make([]gff.Field, 0, 4)
+		if d.strref != noStrref {
+			fields = append(fields, gff.NewField("STRREF", gff.DWordValue(d.strref)))
+		} else {
+			fields = append(fields, gff.NewField("NAME", gff.StringValue(d.name)))
+		}
+		fields = append(fields, gff.NewField("RESREF", gff.ResRefValue(d.resref)))
+		if d.creature {
+			fields = append(fields, gff.NewField("CR", gff.FloatValue(d.cr)))
+			fields = append(fields, gff.NewField("FACTION", gff.StringValue(d.faction)))
+		}
+		list = append(list, gff.Struct{Fields: fields})
+	}
+	return list
+}
+
+func structHasField(s gff.Struct, label string) bool {
+	for _, field := range s.Fields {
+		if field.Label == label {
+			return true
+		}
+	}
+	return false
+}

internal/pipeline/palette_test.go

@@ -0,0 +1,191 @@
+package pipeline
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+const paletteTestSkeleton = `{
+  "file_type": "ITP ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 4294967295,
+    "fields": [
+      {
+        "label": "MAIN",
+        "type": "List",
+        "value": [
+          {
+            "struct_type": 0,
+            "fields": [
+              {"label": "STRREF", "type": "DWord", "value": 500},
+              {
+                "label": "LIST",
+                "type": "List",
+                "value": [
+                  {
+                    "struct_type": 0,
+                    "fields": [
+                      {"label": "STRREF", "type": "DWord", "value": 6699},
+                      {"label": "ID", "type": "Byte", "value": 23},
+                      {
+                        "label": "LIST",
+                        "type": "List",
+                        "value": [
+                          {
+                            "struct_type": 0,
+                            "fields": [
+                              {"label": "NAME", "type": "CExoString", "value": "Stale Junk"},
+                              {"label": "RESREF", "type": "ResRef", "value": "stalejunk"}
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  },
+                  {
+                    "struct_type": 0,
+                    "fields": [
+                      {"label": "STRREF", "type": "DWord", "value": 6753},
+                      {"label": "ID", "type": "Byte", "value": 24}
+                    ]
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}
+`
+
+func paletteTestItem(resref, name string, paletteID int) string {
+	return `{
+  "file_type": "UTI ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 4294967295,
+    "fields": [
+      {"label": "TemplateResRef", "type": "ResRef", "value": "` + resref + `"},
+      {
+        "label": "LocalizedName",
+        "type": "CExoLocString",
+        "value": {"string_ref": 4294967295, "entries": [{"id": 0, "value": "` + name + `"}]}
+      },
+      {"label": "PaletteID", "type": "Byte", "value": ` + itoa(paletteID) + `}
+    ]
+  }
+}
+`
+}
+
+func itoa(v int) string {
+	data, _ := json.Marshal(v)
+	return string(data)
+}
+
+func TestBuildProjectsPaletteDescriptorsAndExtractSkipsThem(t *testing.T) {
+	root := t.TempDir()
+	mustMkdir(t, filepath.Join(root, "src", "module"))
+	mustMkdir(t, filepath.Join(root, "src", "palettes"))
+	mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
+	mustMkdir(t, filepath.Join(root, "build"))
+
+	mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
+  "module": {"name": "Test Module", "resref": "testmod"},
+  "paths": {"source": "src", "build": "build"}
+}
+`)
+	mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
+  "file_type": "IFO ",
+  "file_version": "V3.2",
+  "root": {"struct_type": 4294967295, "fields": [{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}]}
+}
+`)
+	mustWriteFile(t, filepath.Join(root, "src", "palettes", "itempalcus.itp.json"), paletteTestSkeleton)
+	mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_b.uti.json"), paletteTestItem("i_b", "Bravo Item", 23))
+	mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_a.uti.json"), paletteTestItem("i_a", "Alpha Item", 23))
+	mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_hidden.uti.json"), paletteTestItem("i_hidden", "Hidden Item", 255))
+
+	p, err := project.Load(root)
+	if err != nil {
+		t.Fatalf("load project: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+	if _, err := BuildModule(p); err != nil {
+		t.Fatalf("build: %v", err)
+	}
+
+	archiveFile, err := os.Open(p.ModuleArchivePath())
+	if err != nil {
+		t.Fatalf("open module archive: %v", err)
+	}
+	defer archiveFile.Close()
+	archive, err := erf.Read(archiveFile)
+	if err != nil {
+		t.Fatalf("read module archive: %v", err)
+	}
+
+	var palette *gff.Document
+	for _, resource := range archive.Resources {
+		if resource.Name == "itempalcus" {
+			document, err := gff.Read(bytes.NewReader(resource.Data))
+			if err != nil {
+				t.Fatalf("decode itempalcus: %v", err)
+			}
+			palette = &document
+		}
+	}
+	if palette == nil {
+		t.Fatal("itempalcus missing from built module")
+	}
+
+	canonical, err := json.Marshal(palette)
+	if err != nil {
+		t.Fatalf("marshal palette: %v", err)
+	}
+	text := string(canonical)
+	if bytes.Contains(canonical, []byte("stalejunk")) {
+		t.Fatalf("stale descriptor survived projection: %s", text)
+	}
+	for _, resref := range []string{"i_a", "i_b"} {
+		if !bytes.Contains(canonical, []byte(resref)) {
+			t.Fatalf("descriptor %s missing from projection: %s", resref, text)
+		}
+	}
+	if bytes.Contains(canonical, []byte("i_hidden")) {
+		t.Fatalf("PaletteID 255 blueprint leaked into projection: %s", text)
+	}
+	if a, b := bytes.Index(canonical, []byte("i_a")), bytes.Index(canonical, []byte("i_b")); a > b {
+		t.Fatalf("descriptors not name-sorted: %s", text)
+	}
+
+	if _, err := Compare(p); err != nil {
+		t.Fatalf("compare after build: %v", err)
+	}
+
+	// Extraction must never write palcus files back into source.
+	if err := os.Remove(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); err != nil {
+		t.Fatalf("remove skeleton: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("rescan: %v", err)
+	}
+	if _, err := Extract(p); err != nil {
+		t.Fatalf("extract: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); !os.IsNotExist(err) {
+		t.Fatalf("extract wrote palcus file back (stat err: %v)", err)
+	}
+}

internal/topdata/native.go

@@ -410,6 +410,9 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
 	if err != nil {
 		return BuildResult{}, err
 	}
+	if err := loadStandaloneTLKStrings(sourceDir, compiler); err != nil {
+		return BuildResult{}, err
+	}
 
 	output2DA := compiled2DAOutputDir(p)
 	outputTLK := compiledTLKOutputDir(p)

internal/topdata/tlk_native.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"golang.org/x/text/encoding/charmap"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -74,6 +75,16 @@ type tlkEntryData struct {
 	SoundLength float32
 }
 
+type standaloneTLKDocument struct {
+	Schema     string `yaml:"schema"`
+	BaseStrref int    `yaml:"base_strref"`
+	Strings    []struct {
+		Key  string `yaml:"key"`
+		Text string `yaml:"text"`
+		ID   *int   `yaml:"id"`
+	} `yaml:"strings"`
+}
+
 type tlkStateDocument struct {
 	Version  int                        `json:"version"`
 	Language string                     `json:"language"`
@@ -98,6 +109,7 @@ type tlkCompiler struct {
 	active       map[string]tlkEntryData
 	activeKeys   map[string]struct{}
 	reservedByID map[int]string
+	pinnedByID   map[int]string
 	nextID       int
 }
 
@@ -147,6 +159,7 @@ func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, erro
 		active:       map[string]tlkEntryData{},
 		activeKeys:   map[string]struct{}{},
 		reservedByID: reserved,
+		pinnedByID:   map[int]string{},
 		nextID:       nextID,
 	}
 	if legacy != nil {
@@ -157,6 +170,42 @@ func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, erro
 	return compiler, nil
 }
 
+func loadStandaloneTLKStrings(sourceDir string, compiler *tlkCompiler) error {
+	path := filepath.Join(sourceDir, "tlk", "custom.tlk.yml")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read %s: %w", path, err)
+	}
+	var document standaloneTLKDocument
+	if err := yaml.Unmarshal(raw, &document); err != nil {
+		return fmt.Errorf("parse %s: %w", path, err)
+	}
+	if document.Schema != "sow-topdata/tlk/v1" {
+		return fmt.Errorf("%s: unsupported schema %q", path, document.Schema)
+	}
+	if document.BaseStrref != customTLKBase {
+		return fmt.Errorf("%s: base_strref must be %d", path, customTLKBase)
+	}
+	seen := map[string]struct{}{}
+	for index, entry := range document.Strings {
+		entry.Key = strings.TrimSpace(entry.Key)
+		if entry.Key == "" || entry.Text == "" || entry.ID == nil {
+			return fmt.Errorf("%s: strings[%d] requires key, text, and id", path, index)
+		}
+		if _, ok := seen[entry.Key]; ok {
+			return fmt.Errorf("%s: duplicate string key %q", path, entry.Key)
+		}
+		seen[entry.Key] = struct{}{}
+		if err := compiler.registerInlineAtID(entry.Key, *entry.ID, tlkEntryData{Text: entry.Text}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func loadBaseDialogData(path string) (*legacyTLKData, error) {
 	if _, err := os.Stat(path); err != nil {
 		if os.IsNotExist(err) {
@@ -329,6 +378,33 @@ func (c *tlkCompiler) registerInline(key string, entry tlkEntryData) (tlkCompile
 	}, nil
 }
 
+func (c *tlkCompiler) registerInlineAtID(key string, id int, entry tlkEntryData) error {
+	if id < 0 {
+		return fmt.Errorf("TLK key %q has negative id %d", key, id)
+	}
+	// The pin is authoritative over the per-machine .tlk_state.json cache: a
+	// stale mapping that dynamically grabbed this id on an older build must
+	// yield so the pinned key can take it. Only a genuine clash between two
+	// pins in custom.tlk.yml is an author error.
+	if owner, ok := c.pinnedByID[id]; ok && owner != key {
+		return fmt.Errorf("TLK id %d is pinned by both %q and %q", id, owner, key)
+	}
+	if mapping, ok := c.state.Entries[key]; ok && mapping.ID != id {
+		// This key held a different cached id; release it so the pin wins.
+		if c.reservedByID[mapping.ID] == key {
+			delete(c.reservedByID, mapping.ID)
+		}
+	}
+	if owner, ok := c.reservedByID[id]; ok && owner != key {
+		// Evict the stale owner; it gets a fresh id when next made active.
+		delete(c.state.Entries, owner)
+	}
+	c.pinnedByID[id] = key
+	c.state.Entries[key] = tlkStateMapping{ID: id}
+	c.reservedByID[id] = key
+	return c.markActive(key, entry)
+}
+
 func (c *tlkCompiler) customStrrefForKey(key string) int {
 	return customTLKBase + c.state.Entries[key].ID
 }
@@ -338,6 +414,7 @@ func (c *tlkCompiler) markActive(key string, entry tlkEntryData) error {
 	if !ok {
 		mapping = tlkStateMapping{ID: c.allocateID(), Retired: false}
 		c.state.Entries[key] = mapping
+		c.reservedByID[mapping.ID] = key
 	}
 	existing, ok := c.active[key]
 	if ok && existing != entry {
@@ -351,9 +428,13 @@ func (c *tlkCompiler) markActive(key string, entry tlkEntryData) error {
 }
 
 func (c *tlkCompiler) allocateID() int {
-	id := c.nextID
-	c.nextID++
-	return id
+	for {
+		id := c.nextID
+		c.nextID++
+		if _, reserved := c.reservedByID[id]; !reserved {
+			return id
+		}
+	}
 }
 
 func (c *tlkCompiler) finish(outputDir, tlkName string) (int, error) {

internal/topdata/top_package.go

@@ -2,6 +2,7 @@ package topdata
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -13,6 +14,7 @@ import (
 	"time"
 
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
 )
 
@@ -215,10 +217,15 @@ func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]er
 			if err != nil {
 				return err
 			}
-			if skipTopPackageAsset(rel) {
-				return nil
+			var resource erf.Resource
+			if strings.HasSuffix(strings.ToLower(rel), ".itp.json") {
+				resource, err = topPackageITPResourceFromJSON(path)
+			} else {
+				if skipTopPackageAsset(rel) {
+					return nil
+				}
+				resource, err = topPackageResourceFromPath(path)
 			}
-			resource, err := topPackageResourceFromPath(path)
 			if err != nil {
 				return err
 			}
@@ -251,6 +258,27 @@ func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]er
 	return resources, assetFiles, nil
 }
 
+func topPackageITPResourceFromJSON(path string) (erf.Resource, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return erf.Resource{}, fmt.Errorf("read %s: %w", path, err)
+	}
+	var document gff.Document
+	if err := json.Unmarshal(raw, &document); err != nil {
+		return erf.Resource{}, fmt.Errorf("parse %s: %w", path, err)
+	}
+	if document.FileType != "ITP " {
+		return erf.Resource{}, fmt.Errorf("%s: file_type must be ITP", path)
+	}
+	var payload bytes.Buffer
+	if err := gff.Write(&payload, document); err != nil {
+		return erf.Resource{}, fmt.Errorf("compile %s: %w", path, err)
+	}
+	resourceType, _ := erf.HAKResourceTypeForExtension("itp")
+	name := strings.TrimSuffix(filepath.Base(path), ".itp.json")
+	return erf.Resource{Name: strings.ToLower(name), Type: resourceType, Data: payload.Bytes(), Size: int64(payload.Len())}, nil
+}
+
 func shouldSkipTopDataSourceDir(path string, skipDirs map[string]struct{}) bool {
 	_, ok := skipDirs[filepath.Clean(path)]
 	return ok

internal/topdata/topdata_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
 )
 
@@ -2396,6 +2397,107 @@ func TestBuildUsesNativeModeForInlineTLKAndWritesState(t *testing.T) {
 	}
 }
 
+func TestBuildNativeCompilesStandaloneTLKStrings(t *testing.T) {
+	root := testProjectRoot(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
+	mkdirAll(t, filepath.Join(root, "topdata", "tlk"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
+	  "output": "feat.2da",
+	  "columns": ["LABEL", "FEAT", "DESCRIPTION"],
+	  "rows": [{
+	    "id": 0,
+	    "key": "feat:test",
+	    "LABEL": "TEST_LABEL",
+	    "FEAT": {"tlk": {"key": "feat:test.name", "text": "Test Feat"}},
+	    "DESCRIPTION": "****"
+	  }]
+	}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "tlk", "custom.tlk.yml"), `schema: sow-topdata/tlk/v1
+base_strref: 16777216
+strings:
+  - key: palette.creatures.npcs
+    text: NPCs
+    id: 50
+`)
+
+	result, err := BuildNative(testProject(root), nil)
+	if err != nil {
+		t.Fatalf("BuildNative failed: %v", err)
+	}
+	stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile))
+	if err != nil {
+		t.Fatalf("read tlk state: %v", err)
+	}
+	var state tlkStateDocument
+	if err := json.Unmarshal(stateRaw, &state); err != nil {
+		t.Fatalf("parse tlk state: %v", err)
+	}
+	mapping, ok := state.Entries["palette.creatures.npcs"]
+	if !ok || mapping.Retired {
+		t.Fatalf("expected active standalone TLK mapping, got %#v", mapping)
+	}
+	if got := state.Entries["feat:test.name"].ID; got != 0 || mapping.ID != 50 {
+		t.Fatalf("standalone strings must keep pinned ids without shifting dataset refs, got dataset=%d standalone=%d", got, mapping.ID)
+	}
+	tlkRaw, err := os.ReadFile(filepath.Join(result.OutputTLKDir, defaultTLKName))
+	if err != nil {
+		t.Fatalf("read compiled tlk: %v", err)
+	}
+	if !bytes.Contains(tlkRaw, []byte("NPCs")) {
+		t.Fatalf("compiled TLK does not contain standalone text")
+	}
+}
+
+func TestBuildStandaloneTLKPinEvictsStaleStateOwner(t *testing.T) {
+	root := testProjectRoot(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
+	mkdirAll(t, filepath.Join(root, "topdata", "tlk"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
+	  "output": "feat.2da",
+	  "columns": ["LABEL", "FEAT", "DESCRIPTION"],
+	  "rows": [{
+	    "id": 0,
+	    "key": "feat:test",
+	    "LABEL": "TEST_LABEL",
+	    "FEAT": {"tlk": {"key": "feat:test.name", "text": "Test Feat"}},
+	    "DESCRIPTION": "****"
+	  }]
+	}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "tlk", "custom.tlk.yml"), `schema: sow-topdata/tlk/v1
+base_strref: 16777216
+strings:
+  - key: sow.module.name
+    text: Shadows Over Westgate
+    id: 50
+`)
+	// Simulate a per-machine state that predates the pinned taxonomy: the feat
+	// ref dynamically grabbed id 50 on an older build, exactly where the pin
+	// now lives. The build must self-heal instead of failing.
+	writeFile(t, filepath.Join(root, "topdata", tlkStateFile),
+		`{"version":1,"language":"en","entries":{"feat:test.name":{"id":50}}}`+"\n")
+
+	if _, err := BuildNative(testProject(root), nil); err != nil {
+		t.Fatalf("BuildNative failed on stale pin collision: %v", err)
+	}
+
+	stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile))
+	if err != nil {
+		t.Fatalf("read tlk state: %v", err)
+	}
+	var state tlkStateDocument
+	if err := json.Unmarshal(stateRaw, &state); err != nil {
+		t.Fatalf("parse tlk state: %v", err)
+	}
+	if state.Entries["sow.module.name"].ID != 50 {
+		t.Fatalf("pin must own id 50, got %#v", state.Entries["sow.module.name"])
+	}
+	if got := state.Entries["feat:test.name"].ID; got == 50 {
+		t.Fatalf("stale feat ref should have been reallocated off id 50, got %d", got)
+	}
+}
+
 func TestBuildPreservesTLKStateAcrossTextChanges(t *testing.T) {
 	root := testProjectRoot(t)
 	mkdirAll(t, filepath.Join(root, "topdata", "data", "skills"))
@@ -9354,6 +9456,60 @@ func TestBuildAndPackageIncludesCompiled2DAAndTopAssets(t *testing.T) {
 	}
 }
 
+func TestBuildAndPackageCompilesITPJSONAssets(t *testing.T) {
+	root := testProjectRoot(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))
+	mkdirAll(t, filepath.Join(root, "topdata", "assets", "palette"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{
+  "output": "repadjust.2da",
+  "columns": ["Label"],
+  "rows": [{"id": 0, "Label": "TEST_LABEL"}]
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "assets", "palette", "creaturepal.itp.json"), `{
+  "file_type": "ITP ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 4294967295,
+    "fields": [
+      {"label": "MAIN", "type": "List", "value": []},
+      {"label": "RESTYPE", "type": "Word", "value": 2027},
+      {"label": "NEXT_USEABLE_ID", "type": "Byte", "value": 51}
+    ]
+  }
+}`+"\n")
+
+	proj := testProject(root)
+	proj.Config.TopData.ReferenceBuilder = ""
+	result, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("BuildAndPackage failed: %v", err)
+	}
+	input, err := os.Open(result.OutputHAKPath)
+	if err != nil {
+		t.Fatalf("open hak: %v", err)
+	}
+	defer input.Close()
+	archive, err := erf.Read(input)
+	if err != nil {
+		t.Fatalf("read hak: %v", err)
+	}
+	for _, resource := range archive.Resources {
+		if resource.Name != "creaturepal" || resource.Type != 0x07EE {
+			continue
+		}
+		document, err := gff.Read(bytes.NewReader(resource.Data))
+		if err != nil {
+			t.Fatalf("read compiled ITP: %v", err)
+		}
+		if document.FileType != "ITP " || document.FileVersion != "V3.2" {
+			t.Fatalf("unexpected compiled ITP header: %#v", document)
+		}
+		return
+	}
+	t.Fatalf("expected creaturepal.itp in top package")
+}
+
 func TestCollectTopPackageResourcesRejectsDuplicateTopAssetKeys(t *testing.T) {
 	root := testProjectRoot(t)
 	mkdirAll(t, filepath.Join(root, ".cache", "2da"))