From 450c3ebc59340219403ad11813c8f136ec88cbd4 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 23 Jul 2026 11:24:49 +0200 Subject: [PATCH] Generate palette projections at module build; never extract them *palcus.itp files are Toolset-generated snapshots: category skeleton plus one descriptor per custom blueprint. Treat them as derived data. Module source now commits only descriptor-free skeletons; build strips any stale descriptors and re-inserts descriptors computed from the blueprints in the source tree (name/STRREF, resref, and CR/faction for creatures), so the DM Creator lists always match what actually ships in the .mod. Compare applies the same projection. Extract skips *palcus.itp entirely, so local Toolset palette churn can no longer leak into module source. Blueprints with PaletteID 255 (hidden) or no PaletteID are omitted, per engine convention. Co-Authored-By: Claude Fable 5 --- internal/pipeline/build.go | 12 +- internal/pipeline/compare.go | 7 + internal/pipeline/extract.go | 7 + internal/pipeline/palette.go | 298 ++++++++++++++++++++++++++++++ internal/pipeline/palette_test.go | 191 +++++++++++++++++++ 5 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 internal/pipeline/palette.go create mode 100644 internal/pipeline/palette_test.go diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 0f13069..33de310 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -593,9 +593,14 @@ func envBool(name string) bool { func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) { var moduleResources []erf.Resource + palettes, err := collectPaletteDescriptors(p) + if err != nil { + return nil, err + } + for _, rel := range p.Inventory.SourceFiles { abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) - resource, err := resourceFromJSON(abs, moduleHakOrder) + resource, err := resourceFromJSON(abs, moduleHakOrder, palettes) if err != nil { return nil, err } @@ -1030,7 +1035,7 @@ func compareResourceKeys(a, b erf.Resource) int { return 0 } -func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) { +func resourceFromJSON(path string, moduleHakOrder []string, palettes paletteProjection) (erf.Resource, error) { name, extension, err := splitSourceName(path) if err != nil { return erf.Resource{}, err @@ -1059,6 +1064,9 @@ func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 { setModuleHAKList(&document, moduleHakOrder) } + if extension == ".itp" && isPaletteProjectionResref(name) { + projectPaletteDocument(&document, palettes[strings.ToLower(name)]) + } var buf bytes.Buffer if err := gff.Write(&buf, document); err != nil { diff --git a/internal/pipeline/compare.go b/internal/pipeline/compare.go index ae74284..ec065c8 100644 --- a/internal/pipeline/compare.go +++ b/internal/pipeline/compare.go @@ -117,6 +117,10 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro if err != nil { return nil, err } + palettes, err := collectPaletteDescriptors(p) + if err != nil { + return nil, err + } for _, rel := range p.Inventory.SourceFiles { abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) @@ -136,6 +140,9 @@ func expectedResources(p *project.Project) (map[string]resourceExpectation, erro if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 { setModuleHAKList(&document, moduleHakOrder) } + if extension == ".itp" && isPaletteProjectionResref(name) { + projectPaletteDocument(&document, palettes[strings.ToLower(name)]) + } canonical, err := json.Marshal(document) if err != nil { diff --git a/internal/pipeline/extract.go b/internal/pipeline/extract.go index 9786c64..a5d00a8 100644 --- a/internal/pipeline/extract.go +++ b/internal/pipeline/extract.go @@ -124,6 +124,13 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma skippedCount++ continue } + // *palcus.itp are Toolset-generated palette projections; the module + // build regenerates them from source blueprints, so extraction never + // writes them back into source. + if ext == "itp" && isPaletteProjectionResref(resource.Name) { + skippedCount++ + continue + } target, data, err := extractedFile(p, resource, ext) if err != nil { diff --git a/internal/pipeline/palette.go b/internal/pipeline/palette.go new file mode 100644 index 0000000..d5d6bb9 --- /dev/null +++ b/internal/pipeline/palette.go @@ -0,0 +1,298 @@ +package pipeline + +// Custom palette projections (*palcus.itp) are generated artifacts: the Toolset +// derives them from the category frameworks plus the blueprints present in the +// module. Crucible reproduces that projection deterministically at build time so +// the module source only carries descriptor-free category skeletons and the +// blueprint files themselves. Extract never writes palcus files back. + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const noStrref = 0xFFFFFFFF + +// hiddenPaletteID suppresses a blueprint from the Custom palette (engine +// convention; see docs in sow-module's palette research notes). +const hiddenPaletteID = 255 + +var paletteFamilyByExtension = map[string]string{ + ".utc": "creaturepalcus", + ".utd": "doorpalcus", + ".ute": "encounterpalcus", + ".uti": "itempalcus", + ".utm": "storepalcus", + ".utp": "placeablepalcus", + ".uts": "soundpalcus", + ".utt": "triggerpalcus", + ".utw": "waypointpalcus", +} + +type paletteDescriptor struct { + name string + strref uint32 + resref string + creature bool + cr float32 + faction string +} + +func (d paletteDescriptor) sortKey() string { + if d.strref != noStrref || d.name == "" { + return strings.ToLower(d.resref) + } + return strings.ToLower(d.name) +} + +// palette resref (e.g. "itempalcus") -> terminal category ID -> descriptors. +type paletteProjection map[string]map[uint8][]paletteDescriptor + +func isPaletteProjectionResref(name string) bool { + return strings.HasSuffix(strings.ToLower(name), "palcus") +} + +func collectPaletteDescriptors(p *project.Project) (paletteProjection, error) { + projection := paletteProjection{} + factions, err := loadFactionNames(p) + if err != nil { + return nil, err + } + + for _, rel := range p.Inventory.SourceFiles { + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + name, extension, err := splitSourceName(abs) + if err != nil { + return nil, err + } + family, ok := paletteFamilyByExtension[extension] + if !ok { + continue + } + + raw, err := os.ReadFile(abs) + if err != nil { + return nil, fmt.Errorf("read %s: %w", abs, err) + } + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + return nil, fmt.Errorf("parse gff json %s: %w", abs, err) + } + + descriptor, paletteID, ok := blueprintDescriptor(document.Root, name, extension, factions) + if !ok { + continue + } + if projection[family] == nil { + projection[family] = map[uint8][]paletteDescriptor{} + } + projection[family][paletteID] = append(projection[family][paletteID], descriptor) + } + + for _, byID := range projection { + for _, descriptors := range byID { + sort.SliceStable(descriptors, func(i, j int) bool { + a, b := descriptors[i], descriptors[j] + if a.sortKey() != b.sortKey() { + return a.sortKey() < b.sortKey() + } + return a.resref < b.resref + }) + } + } + return projection, nil +} + +func blueprintDescriptor(root gff.Struct, fileName, extension string, factions []string) (paletteDescriptor, uint8, bool) { + descriptor := paletteDescriptor{ + strref: noStrref, + resref: strings.ToLower(fileName), + creature: extension == ".utc", + } + paletteID := -1 + + for _, field := range root.Fields { + switch field.Label { + case "PaletteID": + if v, ok := field.Value.(gff.ByteValue); ok { + paletteID = int(v) + } + case "TemplateResRef": + if v, ok := field.Value.(gff.ResRefValue); ok && v != "" { + descriptor.resref = strings.ToLower(string(v)) + } + case "LocalizedName", "LocName", "FirstName": + if v, ok := field.Value.(gff.LocString); ok { + name, strref := locStringLabel(v) + if field.Label == "FirstName" { + descriptor.name = strings.TrimSpace(descriptor.name + " " + name) + if descriptor.strref == noStrref { + descriptor.strref = strref + } + } else { + descriptor.name = name + descriptor.strref = strref + } + } + case "LastName": + if v, ok := field.Value.(gff.LocString); ok { + name, _ := locStringLabel(v) + descriptor.name = strings.TrimSpace(descriptor.name + " " + name) + } + case "ChallengeRating": + if v, ok := field.Value.(gff.FloatValue); ok { + descriptor.cr = float32(v) + } + case "FactionID": + if v, ok := field.Value.(gff.WordValue); ok && int(v) < len(factions) { + descriptor.faction = factions[v] + } + } + } + + if paletteID < 0 || paletteID == hiddenPaletteID { + return paletteDescriptor{}, 0, false + } + return descriptor, uint8(paletteID), true +} + +func locStringLabel(value gff.LocString) (string, uint32) { + if value.StringRef != noStrref { + return "", value.StringRef + } + for _, entry := range value.Entries { + if entry.Value != "" { + return entry.Value, noStrref + } + } + return "", noStrref +} + +func loadFactionNames(p *project.Project) ([]string, error) { + for _, rel := range p.Inventory.SourceFiles { + if !strings.HasSuffix(strings.ToLower(rel), "repute.fac.json") { + continue + } + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + raw, err := os.ReadFile(abs) + if err != nil { + return nil, fmt.Errorf("read %s: %w", abs, err) + } + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + return nil, fmt.Errorf("parse gff json %s: %w", abs, err) + } + var names []string + for _, field := range document.Root.Fields { + if field.Label != "FactionList" { + continue + } + list, ok := field.Value.(gff.ListValue) + if !ok { + continue + } + for _, faction := range list { + name := "" + for _, f := range faction.Fields { + if f.Label == "FactionName" { + if v, ok := f.Value.(gff.StringValue); ok { + name = string(v) + } + } + } + names = append(names, name) + } + } + return names, nil + } + return nil, nil +} + +// projectPaletteDocument strips every blueprint descriptor from the palette +// tree and re-inserts the descriptors derived from module source. Category +// structure (branches, terminal IDs, labels) passes through untouched. +func projectPaletteDocument(document *gff.Document, byID map[uint8][]paletteDescriptor) { + for i, field := range document.Root.Fields { + if field.Label != "MAIN" { + continue + } + if list, ok := field.Value.(gff.ListValue); ok { + document.Root.Fields[i] = gff.NewField("MAIN", projectPaletteNodes(list, byID)) + } + } +} + +func projectPaletteNodes(nodes gff.ListValue, byID map[uint8][]paletteDescriptor) gff.ListValue { + out := make(gff.ListValue, 0, len(nodes)) + for _, node := range nodes { + if structHasField(node, "RESREF") { + continue // blueprint descriptor — regenerated below + } + + terminalID := -1 + fields := make([]gff.Field, 0, len(node.Fields)) + for _, field := range node.Fields { + if field.Label == "LIST" { + continue // rebuilt for terminals, recursed for branches + } + if field.Label == "ID" { + if v, ok := field.Value.(gff.ByteValue); ok { + terminalID = int(v) + } + } + fields = append(fields, field) + } + + switch { + case terminalID >= 0: + if descriptors := byID[uint8(terminalID)]; len(descriptors) > 0 { + fields = append(fields, gff.NewField("LIST", descriptorList(descriptors))) + } + default: + for _, field := range node.Fields { + if field.Label == "LIST" { + if list, ok := field.Value.(gff.ListValue); ok { + fields = append(fields, gff.NewField("LIST", projectPaletteNodes(list, byID))) + } + } + } + } + out = append(out, gff.Struct{Type: node.Type, Fields: fields}) + } + return out +} + +func descriptorList(descriptors []paletteDescriptor) gff.ListValue { + list := make(gff.ListValue, 0, len(descriptors)) + for _, d := range descriptors { + fields := make([]gff.Field, 0, 4) + if d.strref != noStrref { + fields = append(fields, gff.NewField("STRREF", gff.DWordValue(d.strref))) + } else { + fields = append(fields, gff.NewField("NAME", gff.StringValue(d.name))) + } + fields = append(fields, gff.NewField("RESREF", gff.ResRefValue(d.resref))) + if d.creature { + fields = append(fields, gff.NewField("CR", gff.FloatValue(d.cr))) + fields = append(fields, gff.NewField("FACTION", gff.StringValue(d.faction))) + } + list = append(list, gff.Struct{Fields: fields}) + } + return list +} + +func structHasField(s gff.Struct, label string) bool { + for _, field := range s.Fields { + if field.Label == label { + return true + } + } + return false +} diff --git a/internal/pipeline/palette_test.go b/internal/pipeline/palette_test.go new file mode 100644 index 0000000..b55cb84 --- /dev/null +++ b/internal/pipeline/palette_test.go @@ -0,0 +1,191 @@ +package pipeline + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const paletteTestSkeleton = `{ + "file_type": "ITP ", + "file_version": "V3.2", + "root": { + "struct_type": 4294967295, + "fields": [ + { + "label": "MAIN", + "type": "List", + "value": [ + { + "struct_type": 0, + "fields": [ + {"label": "STRREF", "type": "DWord", "value": 500}, + { + "label": "LIST", + "type": "List", + "value": [ + { + "struct_type": 0, + "fields": [ + {"label": "STRREF", "type": "DWord", "value": 6699}, + {"label": "ID", "type": "Byte", "value": 23}, + { + "label": "LIST", + "type": "List", + "value": [ + { + "struct_type": 0, + "fields": [ + {"label": "NAME", "type": "CExoString", "value": "Stale Junk"}, + {"label": "RESREF", "type": "ResRef", "value": "stalejunk"} + ] + } + ] + } + ] + }, + { + "struct_type": 0, + "fields": [ + {"label": "STRREF", "type": "DWord", "value": 6753}, + {"label": "ID", "type": "Byte", "value": 24} + ] + } + ] + } + ] + } + ] + } + ] + } +} +` + +func paletteTestItem(resref, name string, paletteID int) string { + return `{ + "file_type": "UTI ", + "file_version": "V3.2", + "root": { + "struct_type": 4294967295, + "fields": [ + {"label": "TemplateResRef", "type": "ResRef", "value": "` + resref + `"}, + { + "label": "LocalizedName", + "type": "CExoLocString", + "value": {"string_ref": 4294967295, "entries": [{"id": 0, "value": "` + name + `"}]} + }, + {"label": "PaletteID", "type": "Byte", "value": ` + itoa(paletteID) + `} + ] + } +} +` +} + +func itoa(v int) string { + data, _ := json.Marshal(v) + return string(data) +} + +func TestBuildProjectsPaletteDescriptorsAndExtractSkipsThem(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "palettes")) + mustMkdir(t, filepath.Join(root, "src", "blueprints", "items")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test Module", "resref": "testmod"}, + "paths": {"source": "src", "build": "build"} +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": {"struct_type": 4294967295, "fields": [{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}]} +} +`) + mustWriteFile(t, filepath.Join(root, "src", "palettes", "itempalcus.itp.json"), paletteTestSkeleton) + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_b.uti.json"), paletteTestItem("i_b", "Bravo Item", 23)) + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_a.uti.json"), paletteTestItem("i_a", "Alpha Item", 23)) + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "i_hidden.uti.json"), paletteTestItem("i_hidden", "Hidden Item", 255)) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + if _, err := BuildModule(p); err != nil { + t.Fatalf("build: %v", err) + } + + archiveFile, err := os.Open(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("open module archive: %v", err) + } + defer archiveFile.Close() + archive, err := erf.Read(archiveFile) + if err != nil { + t.Fatalf("read module archive: %v", err) + } + + var palette *gff.Document + for _, resource := range archive.Resources { + if resource.Name == "itempalcus" { + document, err := gff.Read(bytes.NewReader(resource.Data)) + if err != nil { + t.Fatalf("decode itempalcus: %v", err) + } + palette = &document + } + } + if palette == nil { + t.Fatal("itempalcus missing from built module") + } + + canonical, err := json.Marshal(palette) + if err != nil { + t.Fatalf("marshal palette: %v", err) + } + text := string(canonical) + if bytes.Contains(canonical, []byte("stalejunk")) { + t.Fatalf("stale descriptor survived projection: %s", text) + } + for _, resref := range []string{"i_a", "i_b"} { + if !bytes.Contains(canonical, []byte(resref)) { + t.Fatalf("descriptor %s missing from projection: %s", resref, text) + } + } + if bytes.Contains(canonical, []byte("i_hidden")) { + t.Fatalf("PaletteID 255 blueprint leaked into projection: %s", text) + } + if a, b := bytes.Index(canonical, []byte("i_a")), bytes.Index(canonical, []byte("i_b")); a > b { + t.Fatalf("descriptors not name-sorted: %s", text) + } + + if _, err := Compare(p); err != nil { + t.Fatalf("compare after build: %v", err) + } + + // Extraction must never write palcus files back into source. + if err := os.Remove(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); err != nil { + t.Fatalf("remove skeleton: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan: %v", err) + } + if _, err := Extract(p); err != nil { + t.Fatalf("extract: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "src", "palettes", "itempalcus.itp.json")); !os.IsNotExist(err) { + t.Fatalf("extract wrote palcus file back (stat err: %v)", err) + } +} -- 2.54.0