fix fragile hak packing (#36)

Reviewed-on: #36
Reviewed-by: xtul <mpiasecki720@protonmail.com>
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

AGENTS.md

@@ -9,6 +9,47 @@ This repo owns the **builder logic:** one Go module
 (`git.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible`
 dispatcher and the `crucible-<name>` binaries.
 
+## What it is / part it serves
+
+Shadows Over Westgate (SoW) is a Neverwinter Nights: Enhanced Edition
+persistent world, split into single-purpose repos. This repo is Crucible, the
+Go build toolkit. The content repos hold game source (module areas, rules
+data, binary assets) and call Crucible to turn that source into artifacts:
+the `.mod` file, 2DA/TLK tables, HAKs, wiki pages. Crucible is the only place
+builder logic lives; content repos only run it through thin wrapper scripts.
+
+## Guidance map
+
+| Where | What |
+|-------|------|
+| `cmd/crucible/`, `cmd/crucible-<name>/` | dispatcher + per-builder shims (thin `main.go` files) |
+| `internal/dispatch/` | the command registry — single source of truth for the command surface |
+| `internal/` (app, pipeline, project, erf, gff, topdata, changelog, validator, depot, menu, buildinfo) | the actual builder logic |
+| `wrappers/` | canonical bootstrap wrappers (`crucible`, `crucible.ps1`) synced to consumer repos; `wrappers/consumers.txt` lists targets |
+| `docs/command-surface.md` | every command, old `nwn-tool` name → new home |
+| `docs/consumer-contract.md` | how consumer repos resolve/pin a Crucible binary |
+| `docs/migration-from-nwn-tool.md` | migration status, what remains |
+| `tests/`, `Makefile`, `flake.nix` | checks, targets, dev shell |
+
+Task routing: adding/changing a command → read `docs/command-surface.md`
+first, then `internal/dispatch`. Changing how consumers get binaries →
+`docs/consumer-contract.md` + `wrappers/`. Release/CI questions → README "CI"
+section and `.gitea/workflows/`.
+
+## How it is used
+
+- Dev loop: `nix develop`, then `make check` / `make build` / `make smoke`
+  (see Commands below).
+- Release: push a `v*` tag. CI uploads cross-built binaries + wrappers to the
+  Gitea release and publishes the `crucible` container image.
+- Consumers (they download released binaries via the wrapper; they never
+  vendor a toolkit):
+  - sow-module — https://git.westgate.pw/ShadowsOverWestgate/sow-module
+  - sow-topdata — https://git.westgate.pw/ShadowsOverWestgate/sow-topdata
+  - sow-assets-manifest — https://git.westgate.pw/ShadowsOverWestgate/sow-assets-manifest
+  - sow-platform (deploys the released image/pins) —
+    https://git.westgate.pw/ShadowsOverWestgate/sow-platform
+
 ## What this repo owns / does not own
 
 Owns: build/extract/validate/compare pipeline, ERF/HAK packing, topdata 2da/tlk

README.md

@@ -80,7 +80,8 @@ This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
 ## CI
 
 PR-first (D7): checks run on pull requests and on push to `main`; the only
-publish event is a `v*` tag (see `sow-docs/runbooks/ci-trigger-standard.md`).
+publish event is a `v*` tag (see `runbooks/ci-trigger-standard.md` in sow-docs,
+https://git.westgate.pw/ShadowsOverWestgate/sow-docs).
 
 - `test.yml` — vet, test, shellcheck, yamllint, binary smoke (PR + main).
 - `test-image.yml` — build the OCI image to prove it compiles (PR + main, no push).

internal/depot/run.go

@@ -143,7 +143,7 @@ func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) stri
 	}
 	shas := shaKeys(shaSizes)
 
-	res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot status:", err)
 		return exitInternal
@@ -185,7 +185,7 @@ func runPush(args []string, stdout, stderr io.Writer, getenv func(string) string
 	}
 	shas := shaKeys(shaSizes)
 
-	res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot push:", err)
 		return exitInternal
@@ -266,7 +266,7 @@ func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) stri
 	}
 	shas := shaKeys(shaSizes)
 
-	res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, shas, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot verify:", err)
 		return exitInternal
@@ -389,7 +389,7 @@ func runPull(args []string, stdout, stderr io.Writer, getenv func(string) string
 
 	// Sweep the source to distinguish "not there" (exit 1, listed) from
 	// "there, download it".
-	res, err := Sweep(context.Background(), backend, toCheck, cfg, io.Discard)
+	res, err := Sweep(context.Background(), backend, toCheck, cfg, stderr)
 	if err != nil {
 		fmt.Fprintln(stderr, "depot pull:", err)
 		return exitInternal

internal/erf/erf.go

@@ -123,6 +123,7 @@ var extensionTypes = map[string]uint16{
 	"mtr": 0x0818,
 	"jpg": 0x081C,
 	"lod": 0x081E,
+	"gif": 0x081F,
 	"png": 0x0820,
 	"lyt": 0x0BB8,
 	"vis": 0x0BB9,
@@ -188,6 +189,7 @@ var typeExtensions = map[uint16]string{
 	0x0818: "mtr",
 	0x081C: "jpg",
 	0x081E: "lod",
+	0x081F: "gif",
 	0x0820: "png",
 	0x0BB8: "lyt",
 	0x0BB9: "vis",

internal/topdata/top_package.go

@@ -211,7 +211,11 @@ func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]er
 			if d.IsDir() {
 				return nil
 			}
-			if strings.HasPrefix(filepath.Base(path), ".") {
+			rel, err := filepath.Rel(assetsDir, path)
+			if err != nil {
+				return err
+			}
+			if skipTopPackageAsset(rel) {
 				return nil
 			}
 			resource, err := topPackageResourceFromPath(path)
@@ -385,6 +389,25 @@ func newestMatchingAutogenOverrideInput(scanRoot string, include []string) (time
 	return newest, newestPath, nil
 }
 
+// skipTopPackageAsset reports whether a file under assets/ is not a HAK
+// resource and must be ignored by both validation and packing: anything in a
+// hidden or underscore-prefixed directory (working dirs like _candidates),
+// hidden files, docs, and any extension that is not a known NWN ResType
+// (erf.extensionTypes is the whitelist). rel is the path relative to assets/.
+func skipTopPackageAsset(rel string) bool {
+	for _, part := range strings.Split(filepath.ToSlash(rel), "/") {
+		if strings.HasPrefix(part, ".") || strings.HasPrefix(part, "_") {
+			return true
+		}
+	}
+	if strings.EqualFold(filepath.Ext(rel), ".md") {
+		return true
+	}
+	ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".")
+	_, ok := erf.HAKResourceTypeForExtension(ext)
+	return !ok
+}
+
 func topPackageResourceFromPath(path string) (erf.Resource, error) {
 	extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
 	resourceType, ok := erf.HAKResourceTypeForExtension(extension)

internal/topdata/topdata.go

@@ -12,7 +12,6 @@ import (
 	"strconv"
 	"strings"
 
-	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
 )
 
@@ -2230,19 +2229,11 @@ func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationRepor
 		if err != nil {
 			return err
 		}
-		if strings.HasPrefix(filepath.Base(path), ".") {
+		if skipTopPackageAsset(rel) {
-			return nil
+			return nil // not a NWN ResType (script, doc, _work dir, ...): never packed, never checked
 		}
 		base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
 		ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
-		if _, ok := erf.HAKResourceTypeForExtension(ext); !ok {
-			report.Diagnostics = append(report.Diagnostics, Diagnostic{
-				Severity: SeverityError,
-				Path:     path,
-				Message:  fmt.Sprintf("unsupported topdata asset HAK resource extension %q", filepath.Ext(path)),
-			})
-			return nil
-		}
 		key := base + "." + ext
 		if previous, ok := seen[key]; ok {
 			report.Diagnostics = append(report.Diagnostics, Diagnostic{

internal/topdata/topdata_test.go

@@ -15115,3 +15115,77 @@ func writeBytes(t *testing.T, path string, content []byte) {
 		t.Fatalf("write %s: %v", path, err)
 	}
 }
+
+func TestSkipTopPackageAsset(t *testing.T) {
+	skipped := []string{
+		"gui/regions/comfyui-generate.sh", // non-NWN extension
+		"gui/regions/_candidates/abyss.opt4.png",
+		"gui/.hidden/banner.png",
+		"gui/.DS_Store",
+		"gui/AGENTS.md",
+		"gui/noextension",
+	}
+	kept := []string{
+		"gui/regions/abyss.png",
+		"2da-src/placeables.2da",
+		"tex/floor01.dds",
+	}
+	for _, rel := range skipped {
+		if !skipTopPackageAsset(rel) {
+			t.Errorf("expected %s to be skipped", rel)
+		}
+	}
+	for _, rel := range kept {
+		if skipTopPackageAsset(rel) {
+			t.Errorf("expected %s to be kept", rel)
+		}
+	}
+}
+
+func TestValidateTopPackageAssetsIgnoresNonNWNFiles(t *testing.T) {
+	dir := t.TempDir()
+	assets := filepath.Join(dir, "assets", "gui", "regions")
+	candidates := filepath.Join(assets, "_candidates")
+	if err := os.MkdirAll(candidates, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for path, content := range map[string]string{
+		filepath.Join(assets, "comfyui-generate.sh"):  "#!/bin/sh",
+		filepath.Join(assets, "abyss.png"):            "png",
+		filepath.Join(candidates, "abyss.opt4.png"):   "png",
+		filepath.Join(candidates, "notes.txt.backup"): "junk",
+	} {
+		if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	dataDir := filepath.Join(dir, "data")
+	if err := os.MkdirAll(dataDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	var report ValidationReport
+	validateTopPackageAssets(dir, dataDir, &report)
+	for _, d := range report.Diagnostics {
+		t.Errorf("unexpected diagnostic: %s: %s", d.Path, d.Message)
+	}
+}
+
+func TestValidateTopPackageAssetsSkipsMarkdown(t *testing.T) {
+	dir := t.TempDir()
+	assets := filepath.Join(dir, "assets", "gui")
+	if err := os.MkdirAll(assets, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(assets, "AGENTS.md"), []byte("docs"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	dataDir := filepath.Join(dir, "data")
+	if err := os.MkdirAll(dataDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	var report ValidationReport
+	validateTopPackageAssets(dir, dataDir, &report)
+	for _, d := range report.Diagnostics {
+		t.Errorf("unexpected diagnostic: %s: %s", d.Path, d.Message)
+	}
+}