From 6b7d5ce69311398709ebe9d99ca8eabb505294d7 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Tue, 28 Jul 2026 18:36:03 +0200 Subject: [PATCH 1/2] fix(erf): align restype numbers with upstream neverwinter.nim Five entries in the restype table disagreed with upstream neverwinter.nim's restype.nim, so HAKs written from those source extensions were labelled with a type the game reads as something else: gff 0x07F7 (2039, upstream bte) -> 0x07F5 (2037) ltr 0x0813 (2067, upstream bak) -> 0x07F4 (2036) jpg 0x081C (2076, upstream tml) -> 0x0821 (2081) dfa -> dft (0x07FD/2045 was already right, only the name was a typo) mdb 0x0816 (2070, upstream xbc) -> removed Upstream registers no restype for mdb, so there is no correct number to give it; keeping it squatting on xbc silently mislabels the resource. It is dropped from AssetExtensions too, which turns .mdb into a loud "unsupported extension" build error instead of a corrupt HAK entry. No asset in the current corpus uses any of these paths. Root cause of the miss: the init() consistency guard only panicked when a number was missing from the reverse map. Its `if canonicalExt == ext { continue }` branch did nothing when the two maps disagreed, so a mismatch was never caught. The guard now panics on disagreement and also checks the reverse direction. Adds a conformance test pinning the shared numbers against upstream, including the four numbers upstream reserves for other extensions, and a test that the two maps are mutual inverses. Refs ShadowsOverWestgate/sow-tools#64 Co-Authored-By: Claude Opus 5 (1M context) --- internal/erf/erf.go | 31 ++++++++++++-------- internal/erf/erf_test.go | 58 +++++++++++++++++++++++++++++++++++-- internal/project/project.go | 2 +- 3 files changed, 76 insertions(+), 15 deletions(-) diff --git a/internal/erf/erf.go b/internal/erf/erf.go index b833ffc..a160d8d 100644 --- a/internal/erf/erf.go +++ b/internal/erf/erf.go @@ -67,6 +67,10 @@ type resourceEntry struct { Size uint32 } +// extensionTypes and typeExtensions must stay exact inverses of each other; +// init() panics if they drift apart. Numbers below 0x0BB8 follow upstream +// neverwinter.nim (neverwinter/restype.nim). The 0x0BB8 and up entries are +// local additions for source formats upstream does not register. var extensionTypes = map[string]uint16{ "res": 0x0000, "bmp": 0x0001, @@ -96,12 +100,13 @@ var extensionTypes = map[string]uint16{ "utt": 0x07F0, "dds": 0x07F1, "uts": 0x07F3, + "ltr": 0x07F4, + "gff": 0x07F5, "fac": 0x07F6, - "gff": 0x07F7, "ute": 0x07F8, "utd": 0x07FA, "utp": 0x07FC, - "dfa": 0x07FD, + "dft": 0x07FD, "gic": 0x07FE, "gui": 0x07FF, "utm": 0x0803, @@ -117,14 +122,12 @@ var extensionTypes = map[string]uint16{ "ndb": 0x0810, "ptm": 0x0811, "ptt": 0x0812, - "ltr": 0x0813, "shd": 0x0815, - "mdb": 0x0816, "mtr": 0x0818, - "jpg": 0x081C, "lod": 0x081E, "gif": 0x081F, "png": 0x0820, + "jpg": 0x0821, "lyt": 0x0BB8, "vis": 0x0BB9, "mdx": 0x0BC0, @@ -162,12 +165,13 @@ var typeExtensions = map[uint16]string{ 0x07F0: "utt", 0x07F1: "dds", 0x07F3: "uts", + 0x07F4: "ltr", + 0x07F5: "gff", 0x07F6: "fac", - 0x07F7: "gff", 0x07F8: "ute", 0x07FA: "utd", 0x07FC: "utp", - 0x07FD: "dfa", + 0x07FD: "dft", 0x07FE: "gic", 0x07FF: "gui", 0x0803: "utm", @@ -183,14 +187,12 @@ var typeExtensions = map[uint16]string{ 0x0810: "ndb", 0x0811: "ptm", 0x0812: "ptt", - 0x0813: "ltr", 0x0815: "shd", - 0x0816: "mdb", 0x0818: "mtr", - 0x081C: "jpg", 0x081E: "lod", 0x081F: "gif", 0x0820: "png", + 0x0821: "jpg", 0x0BB8: "lyt", 0x0BB9: "vis", 0x0BC0: "mdx", @@ -205,8 +207,13 @@ func init() { if !ok { panic(fmt.Sprintf("missing canonical extension for resource type 0x%04X", resourceType)) } - if canonicalExt == ext { - continue + if canonicalExt != ext { + panic(fmt.Sprintf("resource type 0x%04X is %q in extensionTypes but %q in typeExtensions", resourceType, ext, canonicalExt)) + } + } + for resourceType, ext := range typeExtensions { + if registered, ok := extensionTypes[ext]; !ok || registered != resourceType { + panic(fmt.Sprintf("extension %q is 0x%04X in typeExtensions but 0x%04X in extensionTypes (registered=%v)", ext, resourceType, registered, ok)) } } } diff --git a/internal/erf/erf_test.go b/internal/erf/erf_test.go index dd441e3..216d641 100644 --- a/internal/erf/erf_test.go +++ b/internal/erf/erf_test.go @@ -134,8 +134,7 @@ func TestExtensionMappingsSupportModernAssetTypes(t *testing.T) { "mtr": 0x0818, "shd": 0x0815, "txi": 0x07E6, - "jpg": 0x081C, - "mdb": 0x0816, + "jpg": 0x0821, "lyt": 0x0BB8, "vis": 0x0BB9, "mdx": 0x0BC0, @@ -191,3 +190,58 @@ func TestExtensionMappingsSupportAllUTBlueprintTypes(t *testing.T) { } } } + +// TestRestypeTableMatchesUpstream pins the restype numbers Crucible shares with +// neverwinter.nim's restype.nim. The values below are upstream's; a mismatch +// means a HAK we write is mislabelled for the game. +func TestRestypeTableMatchesUpstream(t *testing.T) { + upstream := map[string]uint16{ + "res": 0, "bmp": 1, "mve": 2, "tga": 3, "wav": 4, "plt": 6, "ini": 7, + "bmu": 8, "txt": 10, "mdl": 2002, "nss": 2009, "ncs": 2010, "are": 2012, + "set": 2013, "ifo": 2014, "bic": 2015, "wok": 2016, "2da": 2017, + "tlk": 2018, "txi": 2022, "git": 2023, "uti": 2025, "utc": 2027, + "dlg": 2029, "itp": 2030, "utt": 2032, "dds": 2033, "uts": 2035, + "ltr": 2036, "gff": 2037, "fac": 2038, "ute": 2040, "utd": 2042, + "utp": 2044, "dft": 2045, "gic": 2046, "gui": 2047, "utm": 2051, + "dwk": 2052, "pwk": 2053, "utg": 2055, "jrl": 2056, "utw": 2058, + "ssf": 2060, "hak": 2061, "nwm": 2062, "bik": 2063, "ndb": 2064, + "ptm": 2065, "ptt": 2066, "shd": 2069, "mtr": 2072, "lod": 2078, + "gif": 2079, "png": 2080, "jpg": 2081, + } + for ext, want := range upstream { + got, ok := ResourceTypeForExtension(ext) + if !ok { + t.Errorf("%s: not registered, upstream has %d", ext, want) + continue + } + if got != want { + t.Errorf("%s: registered as %d, upstream has %d", ext, got, want) + } + } + // Extensions upstream registers that Crucible must not reuse for anything else. + reserved := map[uint16]string{2039: "bte", 2067: "bak", 2070: "xbc", 2076: "tml"} + for number, upstreamExt := range reserved { + if ext, ok := ExtensionForResourceType(number); ok { + t.Errorf("%d: registered as %s, upstream reserves it for %s", number, ext, upstreamExt) + } + } +} + +// TestRestypeTablesAreMutualInverses is the check init() is meant to enforce. +func TestRestypeTablesAreMutualInverses(t *testing.T) { + for ext, number := range extensionTypes { + canonical, ok := typeExtensions[number] + if !ok { + t.Errorf("%s: number 0x%04X missing from typeExtensions", ext, number) + continue + } + if canonical != ext { + t.Errorf("%s: maps to 0x%04X, which maps back to %s", ext, number, canonical) + } + } + for number, ext := range typeExtensions { + if got, ok := extensionTypes[ext]; !ok || got != number { + t.Errorf("0x%04X: maps to %s, which maps back to 0x%04X (ok=%v)", number, ext, got, ok) + } + } +} diff --git a/internal/project/project.go b/internal/project/project.go index 5a9dcfa..9334a2f 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -31,7 +31,7 @@ var SourceExtensions = []string{ } var AssetExtensions = []string{ - ".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml", + ".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml", } var BuiltinScriptPrefixes = []string{ -- 2.54.0 From 65d89d4cb87183fed4427efd9562dafc53ad6d72 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Wed, 29 Jul 2026 00:10:50 +0200 Subject: [PATCH 2/2] fix(erf): drop NWN2 formats gr2, wlk and xml from the restype table Follow-up to the mdb removal in the previous commit. Three of the six extensions #64 listed as deliberate local additions are NWN2 formats, not NWN:EE ones, and belong in an NWN:EE tool no more than mdb did. Checked against xoreos (src/aurora/types.h), which covers every Aurora game and so distinguishes the games' tables: gr2 4003, sits in the NWN2 block (MDB2 4000, MDA2 4001, SPT2 4002, GR2 4003, PWC 4008). Granny is NWN2; NWN:EE does not use it. wlk xoreos numbers it 20004 and xml 20003, both above xml kFileTypeMAXArchive in the section headed "Entries for files not found in archives with numerical type IDs / Found in NWN2's ZIP files". Neither has an archive restype in any Aurora game, so Crucible's 0x0BCC and 0x0BCD were invented outright, in a band Aurora does use elsewhere (3022 FSM, 3023 ART). NWN:EE covers those jobs with formats already in the table: wok, pwk and dwk for walkmeshes, and EE's own UI XML loads from ui/ovr rather than from a HAK by restype, which is consistent with upstream neverwinter.nim registering no xml number. The remaining three local additions stay. lyt 3000, vis 3001 and mdx 3008 are real Aurora archive types that NWN1 ships in its own data/*.bif; xoreos gives the same numbers. Upstream simply does not list them. No asset in the corpus uses gr2, wlk or xml, so nothing stops building. Also drops them from AssetExtensions, and records the rule in a comment on the table plus a test, so the next NWN2 format gets caught. Refs ShadowsOverWestgate/sow-tools#64 Co-Authored-By: Claude Opus 5 (1M context) --- internal/erf/erf.go | 21 ++++++++++++--------- internal/erf/erf_test.go | 16 +++++++++++++--- internal/project/project.go | 2 +- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/internal/erf/erf.go b/internal/erf/erf.go index a160d8d..5e207fc 100644 --- a/internal/erf/erf.go +++ b/internal/erf/erf.go @@ -68,9 +68,18 @@ type resourceEntry struct { } // extensionTypes and typeExtensions must stay exact inverses of each other; -// init() panics if they drift apart. Numbers below 0x0BB8 follow upstream -// neverwinter.nim (neverwinter/restype.nim). The 0x0BB8 and up entries are -// local additions for source formats upstream does not register. +// init() panics if they drift apart. +// +// Numbers below 0x0BB8 follow upstream neverwinter.nim +// (neverwinter/restype.nim). The 0x0BB8 and up entries are lyt/vis/mdx: real +// Aurora archive types that NWN1 ships in its own data/*.bif but upstream +// happens not to register. xoreos corroborates those three numbers +// (src/aurora/types.h). +// +// This table is NWN:EE only. Do not add NWN2 formats (mdb, gr2, wlk, xml): +// NWN:EE either gives that number to something else (2070 is xbc here and mdb +// in NWN2) or has no number for it at all, so packing one into a HAK writes a +// resource the game misreads. See the reserved list in erf_test.go. var extensionTypes = map[string]uint16{ "res": 0x0000, "bmp": 0x0001, @@ -131,9 +140,6 @@ var extensionTypes = map[string]uint16{ "lyt": 0x0BB8, "vis": 0x0BB9, "mdx": 0x0BC0, - "wlk": 0x0BCC, - "xml": 0x0BCD, - "gr2": 0x0FA3, } var typeExtensions = map[uint16]string{ @@ -196,9 +202,6 @@ var typeExtensions = map[uint16]string{ 0x0BB8: "lyt", 0x0BB9: "vis", 0x0BC0: "mdx", - 0x0BCC: "wlk", - 0x0BCD: "xml", - 0x0FA3: "gr2", } func init() { diff --git a/internal/erf/erf_test.go b/internal/erf/erf_test.go index 216d641..3634495 100644 --- a/internal/erf/erf_test.go +++ b/internal/erf/erf_test.go @@ -138,9 +138,6 @@ func TestExtensionMappingsSupportModernAssetTypes(t *testing.T) { "lyt": 0x0BB8, "vis": 0x0BB9, "mdx": 0x0BC0, - "xml": 0x0BCD, - "wlk": 0x0BCC, - "gr2": 0x0FA3, } for ext, wantType := range cases { gotType, ok := ResourceTypeForExtension(ext) @@ -227,6 +224,19 @@ func TestRestypeTableMatchesUpstream(t *testing.T) { } } +// TestNWN2FormatsAreNotRegistered keeps NWN2 file formats out of an NWN:EE +// table. mdb and gr2 have Aurora numbers that mean something else (or nothing) +// in NWN:EE; wlk and xml have no archive number in any Aurora game, so the +// values Crucible used for them were invented. Packing any of these into a HAK +// writes a resource the game misreads. +func TestNWN2FormatsAreNotRegistered(t *testing.T) { + for _, ext := range []string{"mdb", "gr2", "wlk", "xml"} { + if number, ok := ResourceTypeForExtension(ext); ok { + t.Errorf("%s is an NWN2 format but is registered as 0x%04X", ext, number) + } + } +} + // TestRestypeTablesAreMutualInverses is the check init() is meant to enforce. func TestRestypeTablesAreMutualInverses(t *testing.T) { for ext, number := range extensionTypes { diff --git a/internal/project/project.go b/internal/project/project.go index 9334a2f..03954ed 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -31,7 +31,7 @@ var SourceExtensions = []string{ } var AssetExtensions = []string{ - ".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml", + ".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".itp", ".jpg", ".lod", ".lyt", ".mdl", ".mdx", ".mtr", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wok", } var BuiltinScriptPrefixes = []string{ -- 2.54.0